koztv-blog-tools 1.2.0 → 1.2.2

package/README.md

@@ -0,0 +1,215 @@
+# koztv-blog-tools
+
+Shared utilities for Telegram-based blog sites. Export posts from Telegram channels, translate with LLM APIs, and generate markdown for static site generators.
+
+## Installation
+
+```bash
+npm install koztv-blog-tools
+```
+
+## Features
+
+- **Telegram Export** — Export full channel history via MTProto (gramjs)
+- **Translation** — Translate posts using OpenAI-compatible APIs (GLM, OpenAI, etc.)
+- **Multi-language** — Export to multiple languages simultaneously
+- **Markdown Generation** — Create markdown files with YAML frontmatter
+- **Media Download** — Download photos, videos, and documents
+- **Incremental Export** — Track processed posts, skip duplicates
+
+## Quick Start
+
+### Export + Translate
+
+```javascript
+const { exportAndTranslate } = require('koztv-blog-tools');
+
+const result = await exportAndTranslate({
+  // Telegram credentials (from https://my.telegram.org)
+  apiId: 12345678,
+  apiHash: 'your_api_hash',
+  session: 'saved_session_string', // from previous auth
+
+  // Target channel
+  channel: '@channelname',
+
+  // Output
+  outputDir: './content/posts',
+  mediaDir: './public/media',
+
+  // Translation (optional)
+  translate: {
+    apiKey: 'your_llm_api_key',
+    apiUrl: 'https://api.openai.com/v1', // or GLM, etc.
+    model: 'gpt-4',
+    sourceLang: 'ru',
+    targetLangs: ['en', 'de'],  // translate to multiple languages
+    keepOriginal: true,         // also save original language
+  },
+
+  // Callbacks
+  onProgress: (msg) => console.log(msg),
+  onSession: (s) => saveSession(s), // save for future use
+});
+
+console.log(`Exported: ${result.exported}, Processed: ${result.processed}`);
+```
+
+### Export Only (no translation)
+
+```javascript
+const { exportTelegramChannel } = require('koztv-blog-tools');
+
+const result = await exportTelegramChannel({
+  apiId: 12345678,
+  apiHash: 'your_api_hash',
+  session: 'saved_session_string',
+  target: '@channelname',
+  outputDir: './export',
+  downloadMedia: true,
+  limit: 100, // optional: limit number of posts
+  since: new Date('2024-01-01'), // optional: filter by date
+});
+```
+
+### Translate Text
+
+```javascript
+const { translateContent, translateTitle } = require('koztv-blog-tools');
+
+const translated = await translateContent('Привет мир', {
+  apiKey: 'your_api_key',
+  apiUrl: 'https://api.openai.com/v1',
+  model: 'gpt-4',
+  sourceLang: 'ru',
+  targetLang: 'en',
+});
+// => "Hello world"
+```
+
+## Authentication
+
+First-time authentication requires QR code login. See `scripts/qr-login.js` in your project:
+
+```javascript
+// Example QR login script
+const { TelegramClient } = require('telegram');
+const { StringSession } = require('telegram/sessions');
+
+const client = new TelegramClient(
+  new StringSession(''),
+  API_ID,
+  API_HASH,
+  { connectionRetries: 5 }
+);
+
+await client.start({
+  phoneNumber: async () => prompt('Phone: '),
+  password: async () => prompt('2FA: '),
+  phoneCode: async () => prompt('Code: '),
+  onError: console.error,
+});
+
+console.log('Session:', client.session.save());
+```
+
+## Output Structure
+
+With multi-language enabled (`targetLangs` + `keepOriginal`):
+
+```
+content/posts/
+  en/
+    my-post-slug/
+      index.md
+  ru/
+    my-post-slug/
+      index.md
+public/media/
+  000123/
+    image1.jpg
+    image2.jpg
+```
+
+Markdown format:
+
+```yaml
+---
+title: "Post Title"
+date: 2024-01-15
+lang: en
+original_link: "https://t.me/channel/123"
+translated_from: "ru"
+---
+
+Post content here...
+
+![](/media/000123/image1.jpg)
+```
+
+## Environment Variables
+
+For GitHub Actions / CI:
+
+```bash
+TELEGRAM_API_ID=12345678
+TELEGRAM_API_HASH=your_api_hash
+TELEGRAM_SESSION=base64_session_string
+TELEGRAM_CHANNEL=@channelname
+
+LLM_API_KEY=your_llm_key
+LLM_API_URL=https://api.openai.com/v1
+LLM_MODEL=gpt-4
+
+TARGET_LANGS=en,de      # comma-separated
+KEEP_ORIGINAL=true      # keep source language
+```
+
+## API Reference
+
+### exportAndTranslate(options)
+
+Main function for export + translation workflow.
+
+**Options:**
+- `apiId`, `apiHash`, `session` — Telegram credentials
+- `channel` — Target channel (@username or ID)
+- `outputDir` — Where to save markdown files
+- `mediaDir` — Where to save media (optional)
+- `limit` — Max posts to export (optional)
+- `since` — Export posts after this date (optional)
+- `downloadMedia` — Download media files (default: true)
+- `translate` — Translation config (optional)
+- `onProgress` — Progress callback
+- `onSession` — Session save callback
+- `processedLog` — Object tracking processed posts
+- `onProcessedLog` — Callback to save processed log
+
+### exportTelegramChannel(options)
+
+Low-level Telegram export.
+
+### translateContent(text, options)
+
+Translate text content.
+
+### translateTitle(title, options)
+
+Translate title (optimized prompt for short text).
+
+### generateEnglishSlug(title, options)
+
+Generate URL-friendly slug from any language title.
+
+## Used By
+
+- [koz.tv](https://koz.tv) — Personal blog with Telegram sync
+- [staskoz.com](https://staskoz.com) — Another blog using this package
+
+## Related
+
+- [k-engine](https://github.com/Koz-TV/k-engine) — Static site generator with multi-language support
+
+## License
+
+MIT

package/dist/index.js

@@ -329,24 +329,36 @@ Do not add any explanations or notes.`
     }
   ];
   const endpoint = apiUrl.endsWith("/") ? `${apiUrl}chat/completions` : `${apiUrl}/chat/completions`;
-  const response = await fetch(endpoint, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Authorization": `Bearer ${apiKey}`
-    },
-    body: JSON.stringify({
-      model,
-      messages,
-      temperature: 0.3
-    })
-  });
-  if (!response.ok) {
+  const maxRetries = 3;
+  let lastError = null;
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    if (attempt > 0) {
+      const delay = 3e3 * Math.pow(2, attempt - 1);
+      await new Promise((r) => setTimeout(r, delay));
+    }
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({
+        model,
+        messages,
+        temperature: 0.3
+      })
+    });
+    if (response.ok) {
+      const data = await response.json();
+      return data.choices[0]?.message?.content || "";
+    }
     const error = await response.text();
-    throw new Error(`API error: ${response.status} - ${error}`);
+    lastError = new Error(`API error: ${response.status} - ${error}`);
+    if (response.status !== 429 && response.status < 500) {
+      throw lastError;
+    }
   }
-  const data = await response.json();
-  return data.choices[0]?.message?.content || "";
+  throw lastError;
 }
 async function translateTitle(title, options) {
   const translated = await translateContent(title, options);
@@ -685,7 +697,7 @@ async function processPost(post, options, exportDir) {
       } catch (error) {
         onProgress?.(`  Error translating to ${targetLang}: ${error.message}`);
       }
-      await new Promise((r) => setTimeout(r, 300));
+      await new Promise((r) => setTimeout(r, 2e3));
     }
   } else {
     languages.push({
@@ -725,9 +737,13 @@ async function processPost(post, options, exportDir) {
         }
       }
       if (mediaFiles.length > 0) {
-        const imageMarkdown = mediaFiles.filter((f) => !f.match(/\.(mp4|mov)$/i)).map((f) => `![](/media/${paddedId}/${f})`).join("\n\n");
-        if (imageMarkdown) {
-          finalBody = finalBody + "\n\n" + imageMarkdown;
+        const images = mediaFiles.filter((f) => !f.match(/\.(mp4|mov|webm|m4v)$/i));
+        const videos = mediaFiles.filter((f) => f.match(/\.(mp4|mov|webm|m4v)$/i));
+        const imageMarkdown = images.map((f) => `![](/media/${paddedId}/${f})`).join("\n\n");
+        const videoMarkdown = videos.map((f) => `<video src="/media/${paddedId}/${f}" controls></video>`).join("\n\n");
+        const mediaMarkdown = [imageMarkdown, videoMarkdown].filter(Boolean).join("\n\n");
+        if (mediaMarkdown) {
+          finalBody = finalBody + "\n\n" + mediaMarkdown;
         }
       }
     }

package/dist/index.mjs

@@ -272,24 +272,36 @@ Do not add any explanations or notes.`
     }
   ];
   const endpoint = apiUrl.endsWith("/") ? `${apiUrl}chat/completions` : `${apiUrl}/chat/completions`;
-  const response = await fetch(endpoint, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Authorization": `Bearer ${apiKey}`
-    },
-    body: JSON.stringify({
-      model,
-      messages,
-      temperature: 0.3
-    })
-  });
-  if (!response.ok) {
+  const maxRetries = 3;
+  let lastError = null;
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    if (attempt > 0) {
+      const delay = 3e3 * Math.pow(2, attempt - 1);
+      await new Promise((r) => setTimeout(r, delay));
+    }
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`
+      },
+      body: JSON.stringify({
+        model,
+        messages,
+        temperature: 0.3
+      })
+    });
+    if (response.ok) {
+      const data = await response.json();
+      return data.choices[0]?.message?.content || "";
+    }
     const error = await response.text();
-    throw new Error(`API error: ${response.status} - ${error}`);
+    lastError = new Error(`API error: ${response.status} - ${error}`);
+    if (response.status !== 429 && response.status < 500) {
+      throw lastError;
+    }
   }
-  const data = await response.json();
-  return data.choices[0]?.message?.content || "";
+  throw lastError;
 }
 async function translateTitle(title, options) {
   const translated = await translateContent(title, options);
@@ -628,7 +640,7 @@ async function processPost(post, options, exportDir) {
       } catch (error) {
         onProgress?.(`  Error translating to ${targetLang}: ${error.message}`);
       }
-      await new Promise((r) => setTimeout(r, 300));
+      await new Promise((r) => setTimeout(r, 2e3));
     }
   } else {
     languages.push({
@@ -668,9 +680,13 @@ async function processPost(post, options, exportDir) {
         }
       }
       if (mediaFiles.length > 0) {
-        const imageMarkdown = mediaFiles.filter((f) => !f.match(/\.(mp4|mov)$/i)).map((f) => `![](/media/${paddedId}/${f})`).join("\n\n");
-        if (imageMarkdown) {
-          finalBody = finalBody + "\n\n" + imageMarkdown;
+        const images = mediaFiles.filter((f) => !f.match(/\.(mp4|mov|webm|m4v)$/i));
+        const videos = mediaFiles.filter((f) => f.match(/\.(mp4|mov|webm|m4v)$/i));
+        const imageMarkdown = images.map((f) => `![](/media/${paddedId}/${f})`).join("\n\n");
+        const videoMarkdown = videos.map((f) => `<video src="/media/${paddedId}/${f}" controls></video>`).join("\n\n");
+        const mediaMarkdown = [imageMarkdown, videoMarkdown].filter(Boolean).join("\n\n");
+        if (mediaMarkdown) {
+          finalBody = finalBody + "\n\n" + mediaMarkdown;
         }
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "koztv-blog-tools",
-  "version": "1.2.0",
+  "version": "1.2.2",
   "description": "Shared utilities for Telegram-based blog sites",
   "main": "dist/index.js",
   "module": "dist/index.mjs",