@aj-archipelago/cortex 1.3.56 → 1.3.57

package/.env.sample

@@ -1 +1,3 @@
-AZURE_OAI_API_KEY=_______________________
+AZURE_OAI_API_KEY=_______________________
+APPTEK_API_KEY=""
+APPTEK_API_ENDPOINT="https://api.apptek.com"

package/config.js

@@ -389,6 +389,29 @@ var config = convict({
                 "maxTokenLength": 131072,
                 "supportsStreaming": true
             },
+            "google-translate": {
+                "type": "GOOGLE-TRANSLATE",
+                "url": "https://translation.googleapis.com/language/translate/v2",
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "requestsPerSecond": 10
+            },
+            "groq-chat": {
+                "type": "GROQ-CHAT",
+                "url": "https://api.groq.com/openai/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{GROQ_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "meta-llama/llama-4-scout-17b-16e-instruct"
+                },
+                "requestsPerSecond": 10,
+                "maxTokenLength": 65536,
+                "maxReturnTokens": 4096,
+                "supportsStreaming": true
+            },
             "claude-35-sonnet-vertex": {
                 "type": "CLAUDE-3-VERTEX",
                 "url": "{{claudeVertexUrl}}",

package/helper-apps/mogrt-handler/.env.example

@@ -0,0 +1,24 @@
+# Server Configuration
+# Port number for the MOGRT handler service
+PORT=7072
+
+# AWS Configuration
+# AWS Region for S3 bucket (default: us-east-1)
+AWS_REGION=us-east-1
+
+# S3 bucket name where MOGRT files and manifests will be stored
+S3_BUCKET_NAME=your-bucket-name
+
+# AWS credentials for S3 access
+AWS_ACCESS_KEY_ID=your-access-key-id
+AWS_SECRET_ACCESS_KEY=your-secret-access-key
+
+# Signed URL Configuration
+# Expiration time for signed URLs in seconds (default: 3600 = 1 hour)
+SIGNED_URL_EXPIRY_SECONDS=3600
+
+# Optional: Node environment (development/production)
+NODE_ENV=development
+
+APPTEK_TOKEN=
+BASE_URL

package/helper-apps/mogrt-handler/README.md

@@ -0,0 +1,166 @@
+# MOGRT Handler Service
+
+A service for managing Motion Graphics Templates (MOGRT) files and their preview GIFs with S3 storage integration.
+
+## Table of Contents
+- [Setup](#setup)
+- [Environment Variables](#environment-variables)
+- [API Documentation](#api-documentation)
+  - [Upload MOGRT Files](#upload-mogrt-files)
+  - [Get Master Manifest](#get-master-manifest)
+  - [Get Individual Manifest](#get-individual-manifest)
+- [File Structure](#file-structure)
+- [Error Handling](#error-handling)
+
+## Setup
+
+1. Install dependencies:
+```bash
+npm install
+```
+
+2. Set up environment variables (see [Environment Variables](#environment-variables) section)
+
+3. Start the server:
+```bash
+npm start
+```
+
+The server will start on port 7072 by default.
+
+## Environment Variables
+
+| Variable | Description | Default | Required |
+|----------|-------------|---------|----------|
+| `PORT` | Server port | 7072 | No |
+| `AWS_REGION` | AWS region for S3 | us-east-1 | No |
+| `S3_BUCKET_NAME` | S3 bucket name for file storage | - | Yes |
+| `AWS_ACCESS_KEY_ID` | AWS access key | - | Yes |
+| `AWS_SECRET_ACCESS_KEY` | AWS secret key | - | Yes |
+| `SIGNED_URL_EXPIRY_SECONDS` | Expiration time for signed URLs | 3600 (1 hour) | No |
+
+## API Documentation
+
+### Upload MOGRT Files
+
+Upload a MOGRT file with its preview GIF.
+
+**Endpoint:** `POST /api/MogrtHandler`
+
+**Content-Type:** `multipart/form-data`
+
+**Required Files:**
+- A `.mogrt` file
+- A `.gif` preview file
+
+**Example Request:**
+```bash
+curl -X POST http://localhost:7072/api/MogrtHandler \
+  -F "mogrt=@/path/to/template.mogrt" \
+  -F "preview=@/path/to/preview.gif"
+```
+
+**Success Response:**
+```json
+{
+  "id": "550e8400-e29b-41d4-a716-446655440000",
+  "mogrtFile": "uploads/550e8400-e29b-41d4-a716-446655440000/template.mogrt",
+  "previewFile": "uploads/550e8400-e29b-41d4-a716-446655440000/preview.gif",
+  "uploadDate": "2025-02-05T14:05:39Z",
+  "mogrtUrl": "https://bucket.s3.amazonaws.com/uploads/550e8400-e29b-41d4-a716-446655440000/template.mogrt?[signed-url-params]",
+  "previewUrl": "https://bucket.s3.amazonaws.com/uploads/550e8400-e29b-41d4-a716-446655440000/preview.gif?[signed-url-params]"
+}
+```
+
+**Error Responses:**
+- `400 Bad Request`: Missing required files or invalid file types
+- `500 Internal Server Error`: Server or S3 error
+
+### Get Master Manifest
+
+Retrieve a list of all uploaded MOGRT files.
+
+**Endpoint:** `GET /api/MogrtHandler`
+
+**Example Request:**
+```bash
+curl http://localhost:7072/api/MogrtHandler
+```
+
+**Success Response:**
+```json
+[
+  {
+    "id": "550e8400-e29b-41d4-a716-446655440000",
+    "mogrtFile": "uploads/550e8400-e29b-41d4-a716-446655440000/template.mogrt",
+    "previewFile": "uploads/550e8400-e29b-41d4-a716-446655440000/preview.gif",
+    "uploadDate": "2025-02-05T14:05:39Z",
+    "mogrtUrl": "https://bucket.s3.amazonaws.com/uploads/550e8400-e29b-41d4-a716-446655440000/template.mogrt?[signed-url-params]",
+    "previewUrl": "https://bucket.s3.amazonaws.com/uploads/550e8400-e29b-41d4-a716-446655440000/preview.gif?[signed-url-params]"
+  }
+]
+```
+
+**Error Response:**
+- `500 Internal Server Error`: Server or S3 error
+
+### Get Individual Manifest
+
+Retrieve information about a specific MOGRT upload.
+
+**Endpoint:** `GET /api/MogrtHandler?manifestId=<uuid>`
+
+**Parameters:**
+- `manifestId` (required): UUID of the upload
+
+**Example Request:**
+```bash
+curl http://localhost:7072/api/MogrtHandler?manifestId=550e8400-e29b-41d4-a716-446655440000
+```
+
+**Success Response:**
+```json
+{
+  "id": "550e8400-e29b-41d4-a716-446655440000",
+  "mogrtFile": "uploads/550e8400-e29b-41d4-a716-446655440000/template.mogrt",
+  "previewFile": "uploads/550e8400-e29b-41d4-a716-446655440000/preview.gif",
+  "uploadDate": "2025-02-05T14:05:39Z",
+  "mogrtUrl": "https://bucket.s3.amazonaws.com/uploads/550e8400-e29b-41d4-a716-446655440000/template.mogrt?[signed-url-params]",
+  "previewUrl": "https://bucket.s3.amazonaws.com/uploads/550e8400-e29b-41d4-a716-446655440000/preview.gif?[signed-url-params]"
+}
+```
+
+**Error Responses:**
+- `404 Not Found`: Manifest not found
+- `500 Internal Server Error`: Server or S3 error
+
+## File Structure
+
+Files are organized in S3 with the following structure:
+
+```
+bucket/
+├── master-manifest.json
+└── uploads/
+    └── <uuid>/
+        ├── template.mogrt
+        ├── preview.gif
+        └── manifest.json
+```
+
+## Error Handling
+
+All endpoints return errors in the following format:
+
+```json
+{
+  "error": "Error message description"
+}
+```
+
+Common error scenarios:
+1. Missing required files in upload
+2. Invalid file types (only .mogrt and .gif allowed)
+3. S3 access or permission issues
+4. Missing or invalid manifest ID
+5. Server configuration errors (missing environment variables)

package/helper-apps/mogrt-handler/glossaryHandler.js

@@ -0,0 +1,218 @@
+import fetch from 'node-fetch';
+import dotenv from 'dotenv';
+import { saveGlossaryId, getGlossaryVersions, getGlossaryVersion } from './s3Handler.js';
+
+dotenv.config();
+
+const APPTEK_BASE_URL = process.env.APPTEK_API_URL || 'https://api.apptek.com/api/v2/glossary';
+const APPTEK_TOKEN = process.env.APPTEK_TOKEN;
+
+export default async function GlossaryHandler(context, req) {
+    const { method, url, body, query, headers } = req;
+    // Use token from header if present, else from env
+    const token = APPTEK_TOKEN;
+    if (!token) {
+        context.res = { status: 401, body: { error: 'Missing x-token or APPTEK_TOKEN' } };
+        return;
+    }
+    try {
+        // List glossaries
+        if (method === 'GET' && url.includes('/list')) {
+            const resp = await fetch(`${APPTEK_BASE_URL}/list`, {
+                method: 'GET',
+                headers: { 'accept': 'application/json', 'x-token': token }
+            });
+            const data = await resp.json();
+            context.res = { status: resp.status, body: data };
+            return;
+        }
+        // Create glossary
+        if (method === 'POST' && url.match(/\/api\/glossary\/[a-z]{2}-[a-z]{2}/)) {
+            const langPair = url.match(/\/api\/glossary\/([a-z]{2}-[a-z]{2})/)[1];
+            body.name = ""
+
+            for (const entry of body.entries) {
+                entry.target_alternatives = [];
+            }
+            const resp = await fetch(`${APPTEK_BASE_URL}/${langPair}`, {
+                method: 'POST',
+                headers: { 'accept': 'application/json', 'x-token': token, 'content-type': 'application/json' },
+                body: JSON.stringify(body)
+            });
+            console.log(resp)
+            const data = await resp.json();
+            
+            // If successful, save the glossary ID to S3 with versioning
+            if (resp.status === 200 && data.glossary_id) {
+                try {
+                    const versionInfo = await saveGlossaryId(data.glossary_id, langPair, name);
+                    // Add version info to the response
+                    data.version = {
+                        versionId: versionInfo.versionId,
+                        key: versionInfo.key
+                    };
+                } catch (s3Error) {
+                    console.error('Error saving glossary ID to S3:', s3Error);
+                    // Don't fail the request if S3 storage fails
+                    data.versioningError = 'Failed to save glossary version to S3';
+                }
+            }
+            
+            context.res = { status: resp.status, body: data };
+            return;
+        }
+        // Delete glossary
+        if (method === 'DELETE' && url.match(/\/api\/glossary\/.+/)) {
+            const glossaryId = url.split('/').pop();
+            console.log(`🗑️ Attempting to delete glossary with ID: ${glossaryId}`);
+            
+            try {
+                const resp = await fetch(`${APPTEK_BASE_URL}/${glossaryId}`, {
+                    method: 'DELETE',
+                    headers: { 'accept': 'application/json', 'x-token': token }
+                });
+                
+                console.log(`📤 Delete request sent, response status: ${resp.status}`);
+                
+                const data = await resp.json().catch(() => {
+                    console.log(`⚠️ No JSON in delete response, using empty object`);
+                    return {};
+                });
+                
+                if (resp.status === 200) {
+                    console.log(`✅ Successfully deleted glossary ${glossaryId}`);
+                } else {
+                    console.error(`❌ Failed to delete glossary ${glossaryId}, status: ${resp.status}`, data);
+                }
+                
+                context.res = { status: resp.status, body: data };
+            } catch (error) {
+                console.error(`❌ Error during glossary deletion: ${error.message}`);
+                context.res = { status: 500, body: { error: `Error deleting glossary: ${error.message}` } };
+            }
+            return;
+        }
+        // Edit glossary: delete then create
+        if (method === 'POST' && url.includes('/edit/')) {
+            const glossaryId = url.split('/edit/').pop();
+            // 1. Delete
+            console.log(`🗑️ Deleting glossary with ID: ${glossaryId} as part of edit operation`);
+            try {
+                const deleteResp = await fetch(`${APPTEK_BASE_URL}/${glossaryId}`, {
+                    method: 'DELETE',
+                    headers: { 'accept': 'application/json', 'x-token': token }
+                });
+                
+                console.log(`📤 Delete request (for edit) sent, response status: ${deleteResp.status}`);
+                
+                if (deleteResp.status === 200) {
+                    console.log(`✅ Successfully deleted glossary ${glossaryId} for edit operation`);
+                } else {
+                    console.warn(`⚠️ Non-200 status when deleting glossary for edit: ${deleteResp.status}`);
+                }
+            } catch (deleteError) {
+                console.error(`❌ Error during glossary deletion for edit: ${deleteError.message}`);
+                // Continue with create even if delete fails
+            }
+            // 2. Create (reuse create logic)
+            const { source_lang_code, target_lang_code, entries } = body;
+            body.name = ""
+            const langPair = `${source_lang_code}-${target_lang_code}`;
+            const resp = await fetch(`${APPTEK_BASE_URL}/${langPair}`, {
+                method: 'POST',
+                headers: { 'accept': 'application/json', 'x-token': token, 'content-type': 'application/json' },
+                body: JSON.stringify(body)
+            });
+            const data = await resp.json();
+            
+            // If successful, save the glossary ID to S3 with versioning
+            if (resp.status === 200 && data.glossary_id) {
+                try {
+                    const versionInfo = await saveGlossaryId(data.glossary_id, langPair);
+                    // Add version info to the response
+                    data.version = {
+                        versionId: versionInfo.versionId,
+                        key: versionInfo.key
+                    };
+                } catch (s3Error) {
+                    console.error('Error saving glossary ID to S3:', s3Error);
+                    // Don't fail the request if S3 storage fails
+                    data.versioningError = 'Failed to save glossary version to S3';
+                }
+            }
+            
+            context.res = { status: resp.status, body: data };
+            return;
+        }
+        // Get versions of a glossary
+        if (method === 'GET' && url.match(/\/api\/glossary\/([a-z]{2}-[a-z]{2})\/versions\/(.*)/)) {
+            const matches = url.match(/\/api\/glossary\/([a-z]{2}-[a-z]{2})\/versions\/(.*)/); 
+            const langPair = matches[1];
+            const glossaryId = matches[2];
+            const name = query.name || '';
+            
+            try {
+                const versions = await getGlossaryVersions(glossaryId, langPair, name);
+                context.res = { status: 200, body: { versions } };
+            } catch (error) {
+                context.res = { status: 500, body: { error: error.message } };
+            }
+            return;
+        }
+        
+        // Get a specific version of a glossary
+        if (method === 'GET' && url.match(/\/api\/glossary\/([a-z]{2}-[a-z]{2})\/version\/(.*)\/(.*)/))
+        {
+            const matches = url.match(/\/api\/glossary\/([a-z]{2}-[a-z]{2})\/version\/(.*)\/(.*)/);
+            const langPair = matches[1];
+            const glossaryId = matches[2];
+            const versionId = matches[3];
+            const name = query.name || '';
+            
+            try {
+                const version = await getGlossaryVersion(glossaryId, langPair, versionId, name);
+                context.res = { status: 200, body: version };
+            } catch (error) {
+                context.res = { status: 500, body: { error: error.message } };
+            }
+            return;
+        }
+        
+        // Get glossary by ID
+        if (method === 'GET' && url.match(/\/api\/glossary\/([^/]+)$/))
+        {
+            const glossaryId = url.match(/\/api\/glossary\/([^/]+)$/)[1];
+            console.log(`📖 Fetching glossary with ID: ${glossaryId}`);
+            
+            try {
+                const resp = await fetch(`${APPTEK_BASE_URL}/${glossaryId}`, {
+                    method: 'GET',
+                    headers: {'x-token': token }
+                });
+                
+                console.log(`📤 Get glossary request sent, response status: ${resp.status}`);
+                
+                const data = await resp.json().catch(() => {
+                    console.log(`⚠️ No JSON in response, using empty object`);
+                    return {};
+                });
+                
+                if (resp.status === 200) {
+                    console.log(`✅ Successfully retrieved glossary ${glossaryId}`);
+                } else {
+                    console.error(`❌ Failed to retrieve glossary ${glossaryId}, status: ${resp.status}`, data);
+                }
+                
+                context.res = { status: resp.status, body: data };
+            } catch (error) {
+                console.error(`❌ Error retrieving glossary: ${error.message}`);
+                context.res = { status: 500, body: { error: `Error retrieving glossary: ${error.message}` } };
+            }
+            return;
+        }
+        
+        context.res = { status: 404, body: { error: 'Not found' } };
+    } catch (error) {
+        context.res = { status: 500, body: { error: error.message } };
+    }
+}

package/helper-apps/mogrt-handler/index.js

@@ -0,0 +1,213 @@
+import { uploadToS3, getManifest, saveManifest, removeFromMasterManifest } from './s3Handler.js';
+import Busboy from 'busboy';
+import { v4 as uuidv4 } from 'uuid';
+import path from 'path';
+
+const ALLOWED_EXTENSIONS = {
+    MOGRT: '.mogrt',
+    PREVIEW: ['.gif', '.mp4']
+};
+
+const validateFiles = (files) => {
+    const hasMogrt = files.some(f => path.extname(f.filename).toLowerCase() === ALLOWED_EXTENSIONS.MOGRT);
+    const hasPreview = files.some(f => ALLOWED_EXTENSIONS.PREVIEW.includes(path.extname(f.filename).toLowerCase()));
+    console.log('Validating files:', hasMogrt, hasPreview);
+    if (!hasMogrt || !hasPreview) {
+        throw new Error('Both MOGRT and preview files (GIF or MP4) are required');
+    }
+};
+
+async function MogrtHandler(context, req) {
+    const { method } = req;
+    const { manifestId } = req.query;
+    const id = req.params?.id;
+
+    try {
+        // GET request to fetch manifest
+        if (method === 'GET') {
+            const manifest = await getManifest(manifestId || 'master');
+            context.res = {
+                status: 200,
+                body: manifest
+            };
+            return;
+        }
+
+        // DELETE request to remove MOGRT item
+        if (method === 'DELETE' && id) {
+            console.log(`Attempting to delete MOGRT with ID: ${id} from manifest: ${manifestId || 'master'}`);
+            try {
+                // Use the new removeFromMasterManifest function
+                const removed = await removeFromMasterManifest(id);
+                
+                // If the item was not found
+                if (!removed) {
+                    console.log(`MOGRT with ID ${id} not found in manifest`);
+                    context.res = {
+                        status: 404,
+                        body: { error: `MOGRT with ID ${id} not found` }
+                    };
+                    return;
+                }
+                
+                console.log(`Successfully deleted MOGRT with ID: ${id}`);
+                
+                context.res = {
+                    status: 200,
+                    body: {
+                        success: true,
+                        message: `MOGRT with ID ${id} successfully deleted`
+                    }
+                };
+                return;
+            } catch (error) {
+                console.error(`Error deleting MOGRT with ID ${id}:`, error);
+                context.res = {
+                    status: 500,
+                    body: { error: `Failed to delete MOGRT: ${error.message}` }
+                };
+                return;
+            }
+        }
+        
+        // POST request to upload files
+        if (method === 'POST') {
+            const files = [];
+            const uploadId = uuidv4();
+            let name = null;
+            console.log('Generated uploadId:', uploadId);
+
+            const busboy = Busboy({ 
+                headers: {
+                    'content-type': req.headers['content-type']
+                },
+                limits: {
+                    fileSize: 50 * 1024 * 1024, // 50MB limit
+                    files: 2 // Expect exactly 2 files
+                }
+            });
+
+            return new Promise((resolve, reject) => {
+                busboy.on('file', (fieldname, file, fileInfo) => {
+                    console.log('Processing file:', fieldname, fileInfo.filename);
+                    const ext = path.extname(fileInfo.filename).toLowerCase();
+                    
+                    if (ext !== ALLOWED_EXTENSIONS.MOGRT && !ALLOWED_EXTENSIONS.PREVIEW.includes(ext)) {
+                        const error = new Error('Invalid file type. Only .mogrt, .gif and .mp4 files are allowed.');
+                        file.resume(); // Drain this file
+                        reject(error);
+                        return;
+                    }
+
+                    const chunks = [];
+                    file.on('data', chunk => {
+                        console.log('Received chunk of size:', chunk.length);
+                        chunks.push(chunk);
+                    });
+
+                    file.on('end', () => {
+                        console.log('File upload complete:', fileInfo.filename);
+                        const buffer = Buffer.concat(chunks);
+                        files.push({
+                            fieldname,
+                            filename: fileInfo.filename,
+                            buffer
+                        });
+                    });
+                });
+
+                busboy.on('field', (fieldname, value) => {
+                    console.log('Received field:', fieldname, value);
+                    if (fieldname === 'name') {
+                        name = value;
+                    }
+                });
+
+                busboy.on('finish', async () => {
+                    console.log('All files processed, total files:', files.length);
+                    try {
+                        validateFiles(files);
+
+                        const uploadPromises = files.map(file => {
+                            const key = `${uploadId}/${file.filename}`;
+                            const ext = path.extname(file.filename).toLowerCase();
+                            let contentType;
+                            if (ext === '.mogrt') {
+                                contentType = 'application/octet-stream';
+                            } else if (ext === '.gif') {
+                                contentType = 'image/gif';
+                            } else if (ext === '.mp4') {
+                                contentType = 'video/mp4';
+                            } else {
+                                contentType = 'application/octet-stream';
+                            }
+                            return uploadToS3(key, file.buffer, contentType);
+                        });
+
+                        const uploadResults = await Promise.all(uploadPromises);
+                        console.log('Upload results:', uploadResults);
+
+                        const manifest = {
+                            id: uploadId,
+                            name: name || uploadId, // Use uploadId as fallback if name not provided
+                            mogrtFile: uploadResults.find(r => path.extname(r.key).toLowerCase() === ALLOWED_EXTENSIONS.MOGRT)?.key,
+                            previewFile: uploadResults.find(r => ALLOWED_EXTENSIONS.PREVIEW.includes(path.extname(r.key).toLowerCase()))?.key,
+                            uploadDate: new Date().toISOString()
+                        };
+
+                        await saveManifest(manifest);
+                        console.log('Manifest saved:', manifest);
+
+                        const masterManifest = await getManifest('master');
+                        context.res = {
+                            status: 200,
+                            body: {
+                                manifest,
+                                masterManifest
+                            }
+                        };
+                        resolve();
+                    } catch (error) {
+                        console.error('Error processing files:', error);
+                        context.res = {
+                            status: 500,
+                            body: { error: error.message }
+                        };
+                        reject(error);
+                    }
+                });
+
+                busboy.on('error', (error) => {
+                    console.error('Busboy error:', error);
+                    reject(error);
+                });
+
+                // Handle the request data
+                try {
+                    if (typeof req.body === 'string') {
+                        busboy.end(Buffer.from(req.body));
+                    } else if (req.body instanceof Buffer) {
+                        busboy.end(req.body);
+                    } else if (req.rawBody) {
+                        busboy.end(req.rawBody);
+                    } else {
+                        req.pipe(busboy);
+                    }
+                } catch (error) {
+                    console.error('Error handling request:', error);
+                    reject(error);
+                }
+            });
+        }
+
+        throw new Error('Method not allowed');
+    } catch (error) {
+        console.error('Handler error:', error);
+        context.res = {
+            status: error.status || 500,
+            body: { error: error.message }
+        };
+    }
+}
+
+export default MogrtHandler;