ahok-skill 1.3.1

package/src/server/server.js

@@ -0,0 +1,215 @@
+"use strict";
+/**
+ *--------------------------------------------------------------------------------
+ *  ______     _          _ _            _  _
+ * |  ____|   | |        | | |          | || |
+ * | |__ _   _| |__   ___| | |_   __   _| || |_
+ * |  __| | | | '_ \ / _ \ | __|  \ \ / /__   _|
+ * | |  | |_| | |_) |  __/ | |_    \ V /   | |
+ * |_|   \__,_|_.__/ \___|_|\__|    \_/    |_|
+ *--------------------------------------------------------------------------------
+ *
+ * @website   -  https:
+ * @github    -  https:
+ * @discord   -  https:
+ *
+ * @author    -  Cavira
+ * @copyright -  2025 Cavira OSS
+ * @version   -  4.0.0
+ *
+ *--------------------------------------------------------------------------------
+ * server.js - Application webserver.
+ *--------------------------------------------------------------------------------
+**/
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+const WebSocket = require('ws');
+const { parse } = require('url');
+function server(config = {}) {
+    const ROUTES = [];
+    const WARES = [];
+    const WS_ROUTES = [];
+    const wss = new WebSocket.Server({ noServer: true });
+    const SERVER = http.createServer((req, res) => {
+        let u = parse(req.url, true);
+        req.query = u.query || {};
+        req.path = u.pathname;
+        req.hostname = (req.headers.host || '').split(':')[0].replace(/[^\w.-]/g, '');
+        req.ip = (req.socket.remoteAddress || '').replace(/[^\w.:]/g, '');
+        res.statusCode = 200;
+        res.status = (x) => {
+            res.statusCode = x;
+            return res;
+        };
+        res.json = (x) => {
+            res.writeHead(res.statusCode || 200, { 'Content-Type': 'application/json' });
+            res.end(JSON.stringify(x));
+        };
+        res.send = (x) => {
+            if (x === undefined || x === null)
+                x = '';
+            if (typeof x === 'object')
+                return res.json(x);
+            res.writeHead(res.statusCode || 200, { 'Content-Type': 'text/plain' });
+            res.end(String(x));
+        };
+        res.set = (k, v) => { res.setHeader(k, v); return res; };
+        let r = matchRoute(req.method.toUpperCase(), req.path);
+        req.params = r ? r.params : {};
+        let fns = [...WARES];
+        fns.push(r ? (req, res, next) => r.handler(req, res, next) : (_req, res) => res.status(404).end('404: Not Found'));
+        let i = 0;
+        let next = () => {
+            if (i < fns.length)
+                fns[i++](req, res, next);
+        };
+        next();
+    });
+    SERVER.on('upgrade', (req, socket, head) => {
+        let u = parse(req.url || '', true);
+        let path = u.pathname;
+        if (!path || path.includes('..') || /[\0-\x1F\x7F]/.test(path)) {
+            socket.destroy();
+            return;
+        }
+        for (let i = 0; i < WS_ROUTES.length; i++) {
+            let r = WS_ROUTES[i];
+            if (r.path === path) {
+                wss.handleUpgrade(req, socket, head, (ws) => {
+                    ws.req = req;
+                    r.handler(ws, req);
+                });
+                return;
+            }
+        }
+        socket.destroy();
+    });
+    const matchRoute = (a, b) => {
+        for (let i = 0; i < ROUTES.length; i++) {
+            let r = ROUTES[i];
+            if (r.method !== a && r.method !== 'ALL')
+                continue;
+            let p = r.path.split('/').filter(Boolean);
+            let u = b.split('/').filter(Boolean);
+            if (p.length !== u.length)
+                continue;
+            let params = {};
+            let matched = true;
+            for (let j = 0; j < p.length; j++) {
+                if (p[j].startsWith(':')) {
+                    params[p[j].slice(1)] = decodeURIComponent(u[j]);
+                }
+                else if (p[j] !== u[j]) {
+                    matched = false;
+                    break;
+                }
+            }
+            if (matched)
+                return { handler: r.handler, params };
+        }
+        return null;
+    };
+    const add = (a, b, c) => { ROUTES.push({ method: a.toUpperCase(), path: b, handler: c }); };
+    const use = (a) => { WARES.push(a); };
+    const listen = (a, b) => { SERVER.setTimeout(10000); SERVER.listen(a, b); };
+    const all = (a, b) => { add('ALL', a, b); };
+    const getRoutes = () => ROUTES.reduce((acc, { method, path }) => ((acc[method] = acc[method] || []).push(path), acc), {});
+    const serverStatic = (endpoint, dir) => {
+        const a = path.resolve(dir);
+        if (!fs.existsSync(a) || !fs.statSync(a).isDirectory()) {
+            console.error(`[STATIC] Directory not found or is not a directory: ${a}`);
+            return (req, res, next) => next();
+        }
+        let b = (endpoint.endsWith('/') ? endpoint : endpoint + '/');
+        return function staticMiddleware(req, res, next) {
+            if (req.method !== 'GET' && req.method !== 'HEAD')
+                return next();
+            if (!req.path.startsWith(b))
+                return next();
+            let c = path.join(a, req.path.substring(b.length));
+            let d = path.relative(a, c);
+            if (!(d && !d.startsWith('..') && !path.isAbsolute(d)))
+                return next();
+            fs.stat(c, (err, stats) => {
+                if (err || !stats.isFile())
+                    return next();
+                res.setHeader('Content-Type', getContentType(c));
+                fs.createReadStream(c).pipe(res);
+            });
+        };
+        function getContentType(a) {
+            switch (path.extname(a).toLowerCase()) {
+                case '.html': return 'text/html';
+                case '.js': return 'text/javascript';
+                case '.css': return 'text/css';
+                case '.json': return 'application/json';
+                case '.txt': return 'text/plain';
+                case '.ico': return 'image/x-icon';
+                case '.png': return 'image/png';
+                case '.webp': return 'image/webp';
+                case '.jpg': return 'image/jpeg';
+                case '.jpeg': return 'image/jpeg';
+                case '.gif': return 'image/gif';
+                case '.svg': return 'image/svg+xml';
+                default: return 'application/octet-stream';
+            }
+        }
+    };
+    use((req, res, next) => {
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS');
+        res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key');
+        if (req.method === 'OPTIONS') return res.status(204).send();
+        next();
+    });
+    use((req, res, next) => {
+        if (req.headers['content-type']?.includes('application/json')) {
+            let d = '';
+            let max = config.max_payload_size || 1_000_000;
+            req.on('data', e => {
+                d += e;
+                if (d.length > max) {
+                    res.status(413).end('Payload Too Large');
+                    req.destroy();
+                }
+            });
+            req.on('end', () => {
+                req.rawBody = d;
+                try {
+                    req.body = JSON.parse(d);
+                }
+                catch {
+                    req.body = null;
+                }
+                next();
+            });
+        }
+        else {
+            next();
+        }
+    });
+    return {
+        use,
+        listen,
+        all,
+        serverStatic,
+        routes: ROUTES,
+        getRoutes,
+        get: (a, b) => add('GET', a, b),
+        post: (a, b) => add('POST', a, b),
+        put: (a, b) => add('PUT', a, b),
+        delete: (a, b) => add('DELETE', a, b),
+        patch: (a, b) => add('PATCH', a, b),
+        options: (a, b) => add('OPTIONS', a, b),
+        head: (a, b) => add('HEAD', a, b),
+        all: (a, b) => add('ALL', a, b),
+        ws: (a, b) => WS_ROUTES.push({ path: a, handler: b })
+    };
+}
+module.exports = server;
+/**
+ *--------------------------------------------------------------------------------
+ * @EOF - End Of File
+ *--------------------------------------------------------------------------------
+**/

package/src/server.ts

@@ -0,0 +1 @@
+import "./server/index.js";

package/src/sources/base.ts

@@ -0,0 +1,257 @@
+/**
+ * base source class for openmemory data sources - production grade
+ * 
+ * features:
+ * - custom exception hierarchy
+ * - logging
+ * - retry logic with exponential backoff
+ * - rate limiting
+ */
+
+// -- exceptions --
+
+export class source_error extends Error {
+    source?: string;
+    cause?: Error;
+
+    constructor(msg: string, source?: string, cause?: Error) {
+        super(source ? `[${source}] ${msg}` : msg);
+        this.name = 'source_error';
+        this.source = source;
+        this.cause = cause;
+    }
+}
+
+export class source_auth_error extends source_error {
+    constructor(msg: string, source?: string, cause?: Error) {
+        super(msg, source, cause);
+        this.name = 'source_auth_error';
+    }
+}
+
+export class source_config_error extends source_error {
+    constructor(msg: string, source?: string, cause?: Error) {
+        super(msg, source, cause);
+        this.name = 'source_config_error';
+    }
+}
+
+export class source_rate_limit_error extends source_error {
+    retry_after?: number;
+
+    constructor(msg: string, retry_after?: number, source?: string) {
+        super(msg, source);
+        this.name = 'source_rate_limit_error';
+        this.retry_after = retry_after;
+    }
+}
+
+export class source_fetch_error extends source_error {
+    constructor(msg: string, source?: string, cause?: Error) {
+        super(msg, source, cause);
+        this.name = 'source_fetch_error';
+    }
+}
+
+// -- types --
+
+export interface source_item {
+    id: string;
+    name: string;
+    type: string;
+    [key: string]: any;
+}
+
+export interface source_content {
+    id: string;
+    name: string;
+    type: string;
+    text: string;
+    data: string | Buffer;
+    meta: Record<string, any>;
+}
+
+export interface source_config {
+    max_retries?: number;
+    requests_per_second?: number;
+    log_level?: 'debug' | 'info' | 'warn' | 'error';
+}
+
+// -- rate limiter --
+
+export class rate_limiter {
+    private rps: number;
+    private tokens: number;
+    private last_update: number;
+
+    constructor(requests_per_second: number = 10) {
+        this.rps = requests_per_second;
+        this.tokens = requests_per_second;
+        this.last_update = Date.now();
+    }
+
+    async acquire(): Promise<void> {
+        const now = Date.now();
+        const elapsed = (now - this.last_update) / 1000;
+        this.tokens = Math.min(this.rps, this.tokens + elapsed * this.rps);
+        this.last_update = now;
+
+        if (this.tokens < 1) {
+            const wait_time = ((1 - this.tokens) / this.rps) * 1000;
+            await new Promise(r => setTimeout(r, wait_time));
+            this.tokens = 0;
+        } else {
+            this.tokens -= 1;
+        }
+    }
+}
+
+// -- retry helper --
+
+export async function with_retry<T>(
+    fn: () => Promise<T>,
+    max_attempts: number = 3,
+    base_delay: number = 1000,
+    max_delay: number = 60000
+): Promise<T> {
+    let last_err: Error | null = null;
+
+    for (let attempt = 0; attempt < max_attempts; attempt++) {
+        try {
+            return await fn();
+        } catch (e: any) {
+            last_err = e;
+
+            if (e instanceof source_auth_error) {
+                throw e; // don't retry auth errors
+            }
+
+            if (attempt < max_attempts - 1) {
+                const delay = e instanceof source_rate_limit_error && e.retry_after
+                    ? e.retry_after * 1000
+                    : Math.min(base_delay * Math.pow(2, attempt), max_delay);
+
+                console.warn(`[retry] attempt ${attempt + 1}/${max_attempts} failed: ${e.message}, retrying in ${delay}ms`);
+                await new Promise(r => setTimeout(r, delay));
+            }
+        }
+    }
+
+    throw last_err;
+}
+
+// -- base source --
+
+export abstract class base_source {
+    name: string = 'base';
+    user_id: string;
+    protected _connected: boolean = false;
+    protected _max_retries: number;
+    protected _rate_limiter: rate_limiter;
+
+    constructor(user_id?: string, config?: source_config) {
+        this.user_id = user_id || 'anonymous';
+        this._max_retries = config?.max_retries || 3;
+        this._rate_limiter = new rate_limiter(config?.requests_per_second || 10);
+    }
+
+    get connected(): boolean {
+        return this._connected;
+    }
+
+    async connect(creds?: Record<string, any>): Promise<boolean> {
+        console.log(`[${this.name}] connecting...`);
+        try {
+            const result = await this._connect(creds || {});
+            this._connected = result;
+            if (result) {
+                console.log(`[${this.name}] connected`);
+            }
+            return result;
+        } catch (e: any) {
+            console.error(`[${this.name}] connection failed: ${e.message}`);
+            throw new source_auth_error(e.message, this.name, e);
+        }
+    }
+
+    async disconnect(): Promise<void> {
+        this._connected = false;
+        console.log(`[${this.name}] disconnected`);
+    }
+
+    async list_items(filters?: Record<string, any>): Promise<source_item[]> {
+        if (!this._connected) {
+            await this.connect();
+        }
+
+        await this._rate_limiter.acquire();
+
+        try {
+            const items = await with_retry(
+                () => this._list_items(filters || {}),
+                this._max_retries
+            );
+            console.log(`[${this.name}] found ${items.length} items`);
+            return items;
+        } catch (e: any) {
+            throw new source_fetch_error(e.message, this.name, e);
+        }
+    }
+
+    async fetch_item(item_id: string): Promise<source_content> {
+        if (!this._connected) {
+            await this.connect();
+        }
+
+        await this._rate_limiter.acquire();
+
+        try {
+            return await with_retry(
+                () => this._fetch_item(item_id),
+                this._max_retries
+            );
+        } catch (e: any) {
+            throw new source_fetch_error(e.message, this.name, e);
+        }
+    }
+
+    async ingest_all(filters?: Record<string, any>): Promise<string[]> {
+        const { ingestDocument } = await import('../ops/ingest');
+
+        const items = await this.list_items(filters);
+        const ids: string[] = [];
+        const errors: { id: string; error: string }[] = [];
+
+        console.log(`[${this.name}] ingesting ${items.length} items...`);
+
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            try {
+                const content = await this.fetch_item(item.id);
+                const result = await ingestDocument(
+                    content.type || 'text',
+                    content.data || content.text || '',
+                    { source: this.name, ...content.meta },
+                    undefined,
+                    this.user_id
+                );
+                ids.push(result.root_memory_id);
+            } catch (e: any) {
+                console.warn(`[${this.name}] failed to ingest ${item.id}: ${e.message}`);
+                errors.push({ id: item.id, error: e.message });
+            }
+        }
+
+        console.log(`[${this.name}] ingested ${ids.length} items, ${errors.length} errors`);
+        return ids;
+    }
+
+    protected _get_env(key: string, default_val?: string): string | undefined {
+        return process.env[key] || default_val;
+    }
+
+    // abstract methods for subclasses
+    protected abstract _connect(creds: Record<string, any>): Promise<boolean>;
+    protected abstract _list_items(filters: Record<string, any>): Promise<source_item[]>;
+    protected abstract _fetch_item(item_id: string): Promise<source_content>;
+}

package/src/sources/github.ts

@@ -0,0 +1,156 @@
+/**
+ * github source for openmemory - production grade
+ * requires: @octokit/rest
+ * env vars: GITHUB_TOKEN
+ */
+
+import { base_source, source_config_error, source_item, source_content } from './base';
+
+export class github_source extends base_source {
+    name = 'github';
+    private octokit: any = null;
+
+    async _connect(creds: Record<string, any>): Promise<boolean> {
+        let Octokit: any;
+        try {
+            Octokit = await import('@octokit/rest').then(m => m.Octokit);
+        } catch {
+            throw new source_config_error('missing deps: npm install @octokit/rest', this.name);
+        }
+
+        const token = creds.token || process.env.GITHUB_TOKEN;
+
+        if (!token) {
+            throw new source_config_error('no credentials: set GITHUB_TOKEN', this.name);
+        }
+
+        this.octokit = new Octokit({ auth: token });
+        return true;
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        if (!filters.repo) {
+            throw new source_config_error('repo is required (format: owner/repo)', this.name);
+        }
+
+        const [owner, repo] = filters.repo.split('/');
+        const path = filters.path?.replace(/^\//, '') || '';
+        const include_issues = filters.include_issues || false;
+
+        const results: source_item[] = [];
+
+        // list files
+        try {
+            const resp = await this.octokit.repos.getContent({ owner, repo, path });
+            const contents = Array.isArray(resp.data) ? resp.data : [resp.data];
+
+            for (const content of contents) {
+                results.push({
+                    id: `${filters.repo}:${content.path}`,
+                    name: content.name,
+                    type: content.type === 'dir' ? 'dir' : content.encoding || 'file',
+                    path: content.path,
+                    size: content.size || 0,
+                    sha: content.sha
+                });
+            }
+        } catch (e: any) {
+            console.warn(`[github] failed to list ${path}: ${e.message}`);
+        }
+
+        // list issues if requested
+        if (include_issues) {
+            try {
+                const resp = await this.octokit.issues.listForRepo({ owner, repo, state: 'all', per_page: 50 });
+
+                for (const issue of resp.data) {
+                    results.push({
+                        id: `${filters.repo}:issue:${issue.number}`,
+                        name: issue.title,
+                        type: 'issue',
+                        number: issue.number,
+                        state: issue.state,
+                        labels: issue.labels.map((l: any) => l.name)
+                    });
+                }
+            } catch (e: any) {
+                console.warn(`[github] failed to list issues: ${e.message}`);
+            }
+        }
+
+        return results;
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        const parts = item_id.split(':');
+        const repo_full = parts[0];
+        const [owner, repo] = repo_full.split('/');
+
+        // issue
+        if (parts.length >= 3 && parts[1] === 'issue') {
+            const issue_num = parseInt(parts[2]);
+            const issue = await this.octokit.issues.get({ owner, repo, issue_number: issue_num });
+
+            const comments = await this.octokit.issues.listComments({ owner, repo, issue_number: issue_num });
+
+            const text_parts = [
+                `# ${issue.data.title}`,
+                `**State:** ${issue.data.state}`,
+                `**Labels:** ${issue.data.labels.map((l: any) => l.name).join(', ')}`,
+                '',
+                issue.data.body || ''
+            ];
+
+            for (const comment of comments.data) {
+                text_parts.push(`\n---\n**${comment.user?.login}:** ${comment.body}`);
+            }
+
+            const text = text_parts.join('\n');
+
+            return {
+                id: item_id,
+                name: issue.data.title,
+                type: 'issue',
+                text,
+                data: text,
+                meta: { source: 'github', repo: repo_full, issue_number: issue_num, state: issue.data.state }
+            };
+        }
+
+        // file
+        const path = parts.slice(1).join(':');
+        const resp = await this.octokit.repos.getContent({ owner, repo, path });
+
+        if (Array.isArray(resp.data)) {
+            const text = resp.data.map((c: any) => `- ${c.path}`).join('\n');
+            return {
+                id: item_id,
+                name: path || repo_full,
+                type: 'directory',
+                text,
+                data: text,
+                meta: { source: 'github', repo: repo_full, path }
+            };
+        }
+
+        const content = resp.data;
+        let text = '';
+        let data: string | Buffer = '';
+
+        if (content.content) {
+            data = Buffer.from(content.content, 'base64');
+            try {
+                text = data.toString('utf-8');
+            } catch { }
+        }
+
+        return {
+            id: item_id,
+            name: content.name,
+            type: content.encoding || 'file',
+            text,
+            data,
+            meta: { source: 'github', repo: repo_full, path: content.path, sha: content.sha, size: content.size }
+        };
+    }
+}