openkbs 0.0.66 → 0.0.70

package/templates/.claude/skills/openkbs/patterns/vectordb-archive.md

@@ -0,0 +1,231 @@
+# VectorDB Archive Pattern
+
+Complete working code for archiving items to long-term memory with semantic search.
+
+## Concept
+
+- **Priority items** (memory_, agent_) - Fast, auto-injected, limited capacity
+- **Archive items** - Unlimited, semantic search, for long-term storage
+
+When priority storage fills up, archive old items to VectorDB.
+
+## actions.js
+
+```javascript
+// Archive items to long-term memory (VectorDB)
+[/<archiveItems>([\s\S]*?)<\/archiveItems>/s, async (match) => {
+    try {
+        const content = match[1].trim();
+        const itemIds = JSON.parse(content);
+
+        if (!Array.isArray(itemIds) || itemIds.length === 0) {
+            throw new Error('Must provide an array of itemIds to archive');
+        }
+
+        const results = [];
+        const embeddingModel = 'text-embedding-3-large';
+        const embeddingDimension = 3072;
+        const timestamp = Date.now();
+
+        for (const itemId of itemIds) {
+            try {
+                // 1. Fetch the original item
+                const originalItem = await openkbs.getItem(itemId);
+                if (!originalItem?.item?.body) {
+                    results.push({ itemId, status: 'error', error: 'Item not found' });
+                    continue;
+                }
+
+                const body = originalItem.item.body;
+                const originalItemType = itemId.split('_')[0];
+
+                // 2. Build embedding text based on item type
+                let embeddingText = '';
+                if (originalItemType === 'memory') {
+                    embeddingText = `${itemId}: ${typeof body.value === 'string' ? body.value : JSON.stringify(body.value)}`;
+                } else if (originalItemType === 'telegram') {
+                    const date = body.date ? new Date(body.date * 1000).toISOString() : '';
+                    embeddingText = `[${date}] ${body.from || 'Unknown'}: ${body.text || ''}`;
+                } else {
+                    embeddingText = `${itemId}: ${JSON.stringify(body)}`;
+                }
+
+                // 3. Create embeddings
+                const { embeddings, totalTokens } = await openkbs.createEmbeddings(embeddingText, embeddingModel);
+
+                // 4. Create archive item with timestamp for uniqueness
+                const archiveItemId = `archive_${timestamp}_${itemId}`;
+                const archiveBody = {
+                    originalItemId: itemId,
+                    originalItemType: originalItemType,
+                    content: body,
+                    archivedAt: new Date().toISOString()
+                };
+
+                await openkbs.items({
+                    action: 'createItem',
+                    itemType: 'archive',
+                    itemId: archiveItemId,
+                    attributes: [
+                        { attrType: 'itemId', attrName: 'itemId', encrypted: false },
+                        { attrType: 'body', attrName: 'body', encrypted: true }
+                    ],
+                    item: { body: await openkbs.encrypt(JSON.stringify(archiveBody)) },
+                    totalTokens,
+                    embeddings: embeddings ? embeddings.slice(0, embeddingDimension) : undefined,
+                    embeddingModel,
+                    embeddingDimension
+                });
+
+                // 5. Delete original item from priority storage
+                await openkbs.deleteItem(itemId);
+
+                results.push({
+                    itemId,
+                    archiveItemId,
+                    status: 'success',
+                    tokens: totalTokens
+                });
+
+            } catch (e) {
+                results.push({ itemId, status: 'error', error: e.message });
+            }
+        }
+
+        const successCount = results.filter(r => r.status === 'success').length;
+        const errorCount = results.filter(r => r.status === 'error').length;
+
+        return {
+            type: "ITEMS_ARCHIVED",
+            summary: `Archived ${successCount} of ${itemIds.length} items (${errorCount} errors)`,
+            results,
+            _meta_actions: ["REQUEST_CHAT_MODEL"]
+        };
+    } catch (e) {
+        return {
+            type: "ARCHIVE_ERROR",
+            error: e.message,
+            _meta_actions: ["REQUEST_CHAT_MODEL"]
+        };
+    }
+}],
+
+// Search long-term archive memory (VectorDB semantic search)
+[/<searchArchive>([\s\S]*?)<\/searchArchive>/s, async (match) => {
+    try {
+        const content = match[1].trim();
+        const data = JSON.parse(content);
+
+        if (!data.query) {
+            throw new Error('Must provide a "query" for semantic search');
+        }
+
+        const topK = data.topK || 10;
+        const minScore = data.minScore || 0;
+
+        // Call VectorDB search
+        const searchResult = await openkbs.items({
+            action: 'searchVectorDBItems',
+            queryText: data.query,
+            topK: topK,
+            minScore: minScore
+        });
+
+        // Format and decrypt results
+        const formattedResults = [];
+
+        for (const item of (searchResult?.items || [])) {
+            try {
+                let parsed = null;
+                if (item.body) {
+                    const decryptedBody = await openkbs.decrypt(item.body);
+                    parsed = JSON.parse(decryptedBody);
+                }
+
+                formattedResults.push({
+                    archiveItemId: item.itemId,
+                    originalItemId: parsed?.originalItemId,
+                    originalItemType: parsed?.originalItemType,
+                    content: parsed?.content,
+                    archivedAt: parsed?.archivedAt,
+                    score: item.score
+                });
+            } catch (e) {
+                formattedResults.push({
+                    archiveItemId: item.itemId,
+                    score: item.score,
+                    error: 'Failed to decrypt: ' + e.message
+                });
+            }
+        }
+
+        return {
+            type: "ARCHIVE_SEARCH_RESULTS",
+            query: data.query,
+            count: formattedResults.length,
+            results: formattedResults,
+            _meta_actions: ["REQUEST_CHAT_MODEL"]
+        };
+    } catch (e) {
+        return {
+            type: "ARCHIVE_SEARCH_ERROR",
+            error: e.message,
+            _meta_actions: ["REQUEST_CHAT_MODEL"]
+        };
+    }
+}]
+```
+
+## settings.json Configuration
+
+```json
+{
+  "embeddingModel": "text-embedding-3-large",
+  "embeddingDimension": 3072,
+  "searchEngine": "VectorDB",
+  "itemTypes": {
+    "archive": {
+      "attributes": [
+        { "attrName": "itemId", "attrType": "itemId", "encrypted": false },
+        { "attrName": "body", "attrType": "body", "encrypted": true }
+      ]
+    }
+  },
+  "options": {
+    "vectorDBMaxTokens": 25000,
+    "vectorDBTopK": 30,
+    "vectorDBMinScore": 90
+  }
+}
+```
+
+## instructions.txt (LLM prompt)
+
+```
+Long-term Memory:
+
+Archive items from priority storage:
+<archiveItems>["memory_old_item1", "memory_old_item2"]</archiveItems>
+
+Search archived memories semantically:
+<searchArchive>{"query": "what did user say about project X", "topK": 10}</searchArchive>
+
+Use archive when:
+- Priority storage is filling up
+- You need to remember things long-term
+- You want semantic search over past conversations
+```
+
+## Key Points
+
+1. **Two-tier storage**:
+   - Priority items: Fast, limited, auto-injected into context
+   - Archive items: Unlimited, semantic search, manual retrieval
+
+2. **Archive flow**: Fetch → Create embeddings → Store in VectorDB → Delete original
+
+3. **Embedding model** must match settings.json configuration
+
+4. **Encrypted storage** - Archive body is encrypted, decrypted on retrieval
+
+5. **Score-based search** - Results sorted by semantic similarity score

package/templates/.claude/skills/openkbs/patterns/video-generation.md

@@ -0,0 +1,145 @@
+# Video Generation Pattern
+
+Complete working code for AI video generation with async polling.
+
+## actions.js
+
+```javascript
+// Create video - returns pending status, requires polling
+[/<createAIVideo>([\s\S]*?)<\/createAIVideo>/s, async (match) => {
+    try {
+        const data = JSON.parse(match[1].trim());
+
+        const videoModel = data.model || 'sora-2';
+        const prompt = data.prompt;
+
+        // Validate seconds (4, 8, or 12 only)
+        const validSeconds = [4, 8, 12];
+        const seconds = validSeconds.includes(data.seconds) ? data.seconds :
+            validSeconds.reduce((prev, curr) =>
+                Math.abs(curr - data.seconds) < Math.abs(prev - data.seconds) ? curr : prev);
+
+        const params = {
+            video_model: videoModel,
+            seconds: seconds
+        };
+
+        // Reference image OR size (not both)
+        if (data.input_reference_url) {
+            params.input_reference_url = data.input_reference_url;
+        } else {
+            const validSizes = ['720x1280', '1280x720'];
+            params.size = validSizes.includes(data.size) ? data.size : '1280x720';
+        }
+
+        const videoData = await openkbs.generateVideo(prompt, params);
+
+        // Video generation is async - check status
+        if (videoData?.[0]?.status === 'pending') {
+            return {
+                type: 'VIDEO_PENDING',
+                data: {
+                    videoId: videoData[0].video_id,
+                    message: 'Video generation in progress. Use continueVideoPolling to check status.'
+                },
+                ...meta
+            };
+        }
+
+        // Immediate completion (rare)
+        if (videoData?.[0]?.video_url) {
+            return {
+                type: 'CHAT_VIDEO',
+                data: { videoUrl: videoData[0].video_url },
+                ...meta
+            };
+        }
+
+        return { error: 'Video generation failed - no response', ...meta };
+    } catch (error) {
+        return { error: error.message || 'Video creation failed', ...meta };
+    }
+}],
+
+// Poll for video completion
+[/<continueVideoPolling>([\s\S]*?)<\/continueVideoPolling>/s, async (match) => {
+    try {
+        const data = JSON.parse(match[1].trim());
+        const videoId = data.videoId;
+
+        const videoData = await openkbs.checkVideoStatus(videoId);
+
+        if (videoData?.[0]?.status === 'completed' && videoData[0].video_url) {
+            return {
+                type: 'CHAT_VIDEO',
+                data: { videoUrl: videoData[0].video_url },
+                ...meta
+            };
+        }
+
+        if (videoData?.[0]?.status === 'pending') {
+            return {
+                type: 'VIDEO_PENDING',
+                data: {
+                    videoId: videoId,
+                    message: 'Video still generating. Continue polling.'
+                },
+                ...meta
+            };
+        }
+
+        if (videoData?.[0]?.status === 'failed') {
+            return { error: 'Video generation failed', ...meta };
+        }
+
+        return { error: 'Unable to get video status', ...meta };
+    } catch (error) {
+        return { error: error.message || 'Failed to check video status', ...meta };
+    }
+}]
+```
+
+## contentRender.js
+
+```javascript
+// In isVisualResult function
+const isVisualResult = (r) => {
+    return (r?.type === 'CHAT_VIDEO' && r?.data?.videoUrl) ||
+           r?.type === 'VIDEO_PENDING';
+};
+
+// In renderVisualResults function
+if (item?.type === 'CHAT_VIDEO' && item?.data?.videoUrl) {
+    return (
+        <div key={`vid-${idx}`} style={{ flex: '1 1 100%' }}>
+            <video
+                src={item.data.videoUrl}
+                controls
+                style={{ width: '100%', maxWidth: 600, borderRadius: 8 }}
+            />
+        </div>
+    );
+}
+```
+
+## instructions.txt (LLM prompt)
+
+```
+To generate a video:
+<createAIVideo>{"prompt": "scene description", "model": "sora-2", "seconds": 8, "size": "1280x720"}</createAIVideo>
+
+Video generation takes time. When you receive VIDEO_PENDING, poll with:
+<continueVideoPolling>{"videoId": "the-video-id"}</continueVideoPolling>
+
+Models: sora-2 (standard), sora-2-pro (higher quality)
+Duration: 4, 8, or 12 seconds
+Size: 1280x720 (landscape), 720x1280 (portrait)
+Optional: input_reference_url for image-to-video
+```
+
+## Key Points
+
+1. **Async by nature** - Video generation returns `VIDEO_PENDING`, not immediate result
+2. **Polling required** - LLM must call `continueVideoPolling` to check completion
+3. **CHAT_VIDEO is core magic** - Renders as video player automatically
+4. **Two commands** - Create and poll are separate for clean flow

package/templates/.claude/skills/openkbs/patterns/web-publishing.md

@@ -0,0 +1,257 @@
+# Web Publishing Pattern
+
+Complete working code for publishing HTML pages to KB storage.
+
+## actions.js
+
+```javascript
+// Publish HTML webpage
+[/<publishWebPage>([\s\S]*?)<\/publishWebPage>/s, async (match) => {
+    try {
+        let htmlContent = match[1].trim();
+
+        // Extract HTML if wrapped in markdown code block
+        const codeBlockMatch = htmlContent.match(/```html\s*([\s\S]*?)\s*```/);
+        if (codeBlockMatch) {
+            htmlContent = codeBlockMatch[1].trim();
+        }
+
+        // Validate HTML content
+        if (!htmlContent || !htmlContent.includes('<html')) {
+            return {
+                type: "PUBLISH_ERROR",
+                error: "No valid HTML content found. Must include <html> tag.",
+                _meta_actions: ["REQUEST_CHAT_MODEL"]
+            };
+        }
+
+        // Fix charset to UTF-8
+        htmlContent = htmlContent.replace(
+            /<meta\s+charset=["']?[^"'>]*["']?\s*\/?>/gi,
+            '<meta charset="UTF-8">'
+        );
+
+        // Extract title for filename
+        const titleMatch = htmlContent.match(/<title>(.*?)<\/title>/i);
+        const title = titleMatch ? titleMatch[1].trim() : 'page';
+
+        // Generate filename from title
+        const timestamp = Date.now();
+        const baseName = title
+            .toLowerCase()
+            .replace(/[^a-z0-9]+/g, '_')
+            .replace(/^_+|_+$/g, '') || 'page';
+        const filename = `${baseName}_${timestamp}.html`;
+
+        // Upload embedded images (optional - for CDN URLs)
+        const cdnUrlRegex = /https:\/\/cdn\.example\.com\/[^\s"'<>]+/g;
+        const cdnUrls = [...new Set(htmlContent.match(cdnUrlRegex) || [])];
+
+        for (const cdnUrl of cdnUrls) {
+            try {
+                // Download and re-upload to permanent storage
+                const fileResponse = await axios.get(cdnUrl, { responseType: 'arraybuffer' });
+                const fileName = cdnUrl.split('/').pop();
+                const fileType = fileResponse.headers['content-type'] || 'image/jpeg';
+
+                const presigned = await openkbs.kb({
+                    action: 'createPresignedURL',
+                    namespace: 'files',
+                    fileName: fileName,
+                    fileType: fileType,
+                    presignedOperation: 'putObject'
+                });
+
+                await axios.put(presigned, fileResponse.data, {
+                    headers: { 'Content-Type': fileType, 'Content-Length': fileResponse.data.length }
+                });
+
+                const permanentUrl = `https://yourdomain.com/media/${fileName}`;
+                htmlContent = htmlContent.split(cdnUrl).join(permanentUrl);
+            } catch (err) {
+                // Continue if image upload fails
+            }
+        }
+
+        // Get presigned URL for HTML upload
+        const presignedUrl = await openkbs.kb({
+            action: 'createPresignedURL',
+            namespace: 'files',
+            fileName: filename,
+            fileType: 'text/html',
+            presignedOperation: 'putObject'
+        });
+
+        // Upload HTML as UTF-8 Buffer
+        const htmlBuffer = Buffer.from(htmlContent, 'utf8');
+        await axios.put(presignedUrl, htmlBuffer, {
+            headers: {
+                'Content-Type': 'text/html',
+                'Content-Length': htmlBuffer.length
+            }
+        });
+
+        const publicUrl = `https://yourdomain.com/media/${filename}`;
+
+        return {
+            type: "WEBSITE_PUBLISHED",
+            filename: filename,
+            url: publicUrl,
+            size: htmlBuffer.length,
+            title: title,
+            message: `Website published at ${publicUrl}`,
+            _meta_actions: ["REQUEST_CHAT_MODEL"]
+        };
+
+    } catch (e) {
+        return {
+            type: "PUBLISH_ERROR",
+            error: e.message,
+            _meta_actions: ["REQUEST_CHAT_MODEL"]
+        };
+    }
+}]
+```
+
+## Helper: Web Publishing Guide
+
+Create `webPublishingGuide.js`:
+
+```javascript
+export const webPublishingGuide = `
+# Web Publishing Guide
+
+## HTML Structure
+- Always include <!DOCTYPE html>
+- Use <meta charset="UTF-8">
+- Set viewport for mobile: <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+## Responsive Design
+- Use flexbox or CSS grid
+- Mobile-first approach
+- Test at 320px, 768px, 1024px widths
+
+## Images
+- Use CDN URLs that will be auto-converted to permanent storage
+- Add alt text for accessibility
+- Use lazy loading: loading="lazy"
+
+## SEO
+- Unique, descriptive <title>
+- Meta description
+- Semantic HTML (header, main, footer, article)
+`;
+
+// Command to read guide
+[/<getWebPublishingGuide\s*\/>/s, async () => {
+    return {
+        type: 'WEB_PUBLISHING_GUIDE',
+        content: webPublishingGuide,
+        ...meta
+    };
+}]
+```
+
+## Example: Complete Landing Page
+
+```html
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Product Launch</title>
+    <style>
+        * { margin: 0; padding: 0; box-sizing: border-box; }
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            line-height: 1.6;
+            color: #333;
+        }
+        .hero {
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            color: white;
+            padding: 80px 20px;
+            text-align: center;
+        }
+        .hero h1 { font-size: 2.5rem; margin-bottom: 1rem; }
+        .hero p { font-size: 1.2rem; opacity: 0.9; max-width: 600px; margin: 0 auto; }
+        .cta {
+            display: inline-block;
+            margin-top: 2rem;
+            padding: 15px 40px;
+            background: white;
+            color: #667eea;
+            text-decoration: none;
+            border-radius: 30px;
+            font-weight: 600;
+        }
+        .features {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+            gap: 30px;
+            padding: 60px 20px;
+            max-width: 1200px;
+            margin: 0 auto;
+        }
+        .feature {
+            text-align: center;
+            padding: 20px;
+        }
+        .feature h3 { margin-bottom: 0.5rem; }
+    </style>
+</head>
+<body>
+    <section class="hero">
+        <h1>Your Product Name</h1>
+        <p>A compelling description of what makes your product amazing.</p>
+        <a href="#" class="cta">Get Started</a>
+    </section>
+    <section class="features">
+        <div class="feature">
+            <h3>Feature One</h3>
+            <p>Description of the first key feature.</p>
+        </div>
+        <div class="feature">
+            <h3>Feature Two</h3>
+            <p>Description of the second key feature.</p>
+        </div>
+        <div class="feature">
+            <h3>Feature Three</h3>
+            <p>Description of the third key feature.</p>
+        </div>
+    </section>
+</body>
+</html>
+```
+
+## instructions.txt (LLM prompt)
+
+```
+Web Publishing:
+
+Publish an HTML page:
+<publishWebPage>
+<!DOCTYPE html>
+<html lang="en">
+...full HTML content...
+</html>
+</publishWebPage>
+
+Get publishing guidelines:
+<getWebPublishingGuide/>
+
+Best practices:
+- Always include DOCTYPE and charset
+- Use inline CSS (no external stylesheets)
+- Images from CDN are auto-converted to permanent URLs
+- Title becomes filename (e.g., "My Page" → my_page_1234567890.html)
+```
+
+## Key Points
+
+1. **Self-contained HTML**: Include all CSS inline, no external dependencies
+2. **UTF-8 encoding**: Always set charset, upload as Buffer
+3. **Auto-filename**: Generated from `<title>` with timestamp
+4. **Image processing**: CDN URLs auto-converted to permanent storage
+5. **CDN delivery**: Pages served via CloudFront at `/media/`

package/templates/.claude/skills/openkbs/reference/backend-sdk.md

@@ -357,8 +357,9 @@ await axios.put(presignedUrl, fileBuffer, {
     }
 });
 
-// Public URL pattern
-const publicUrl = `https://web.file.vpc1.us/files/${openkbs.kbId}/${fileName}`;
+// Public URL pattern (varies by deployment)
+// For whitelabel: https://yourdomain.file.vpc1.us/files/kbId/filename
+// Generic: https://file.openkbs.com/files/kbId/filename
 ```
 
 ## VectorDB (Semantic Search)
@@ -418,6 +419,16 @@ const rates = await openkbs.getExchangeRates('USD');
 // Returns: { EUR: 0.92, GBP: 0.79, ... }
 ```
 
+## Global Objects
+
+These are available globally in all backend handlers (no import needed):
+
+```javascript
+openkbs     // OpenKBS SDK (this document)
+axios       // HTTP client (axios.get, axios.post, etc.)
+crypto      // Node.js crypto module
+```
+
 ## Properties
 
 ```javascript