cognitive-modules-cli 2.2.5 → 2.2.8

package/dist/registry/client.d.ts

@@ -88,14 +88,19 @@ export interface RegistryEntryV2 {
     };
     dependencies: {
         runtime_min: string;
-        modules: string[];
+        modules: Array<{
+            name: string;
+            version?: string;
+            optional?: boolean;
+        }>;
     };
     distribution: {
-        tarball?: string;
-        checksum?: string;
+        tarball: string;
+        checksum: string;
         size_bytes?: number;
         files?: string[];
-        source?: string;
+        signature?: string;
+        signing_key?: string;
     };
     timestamps?: {
         created_at?: string;
@@ -110,6 +115,8 @@ export interface ModuleInfo {
     description: string;
     author: string;
     source: string;
+    tarball?: string;
+    checksum?: string;
     keywords: string[];
     tier?: string;
     namespace?: string;
@@ -127,10 +134,17 @@ export interface SearchResult {
     score: number;
     keywords: string[];
 }
+export declare const DEFAULT_REGISTRY_URL = "https://github.com/Cognary/cognitive/releases/latest/download/cognitive-registry.v2.json";
+export interface RegistryClientOptions {
+    timeoutMs?: number;
+    maxBytes?: number;
+}
 export declare class RegistryClient {
     private registryUrl;
+    private timeoutMs;
+    private maxBytes;
     private cache;
-    constructor(registryUrl?: string);
+    constructor(registryUrl?: string, options?: RegistryClientOptions);
     private parseRegistryResponse;
     /**
      * Generate a unique cache filename based on registry URL
@@ -189,6 +203,7 @@ export declare class RegistryClient {
      */
     getDownloadUrl(moduleName: string): Promise<{
         url: string;
+        checksum?: string;
         isGitHub: boolean;
         githubInfo?: {
             org: string;

package/dist/registry/client.js

@@ -16,26 +16,60 @@ import { createHash } from 'node:crypto';
 // =============================================================================
 // Constants
 // =============================================================================
-const DEFAULT_REGISTRY_URL = 'https://raw.githubusercontent.com/ziel-io/cognitive-modules/main/cognitive-registry.json';
+// "Latest" registry strategy:
+// Prefer GitHub Releases "latest" download, so clients get a coherent set of
+// (index + tarballs) that match an actual published release.
+//
+// This URL is stable across releases:
+// https://github.com/<org>/<repo>/releases/latest/download/cognitive-registry.v2.json
+export const DEFAULT_REGISTRY_URL = 'https://github.com/Cognary/cognitive/releases/latest/download/cognitive-registry.v2.json';
+const FALLBACK_REGISTRY_URL = 'https://raw.githubusercontent.com/Cognary/cognitive/main/cognitive-registry.v2.json';
 const CACHE_DIR = join(homedir(), '.cognitive', 'cache');
 const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
-const REGISTRY_FETCH_TIMEOUT_MS = 10_000; // 10s
-const MAX_REGISTRY_BYTES = 1024 * 1024; // 1MB
+const DEFAULT_REGISTRY_FETCH_TIMEOUT_MS = 10_000; // 10s
+const DEFAULT_MAX_REGISTRY_BYTES = 2 * 1024 * 1024; // 2MB
+const HARD_MAX_REGISTRY_BYTES = 20 * 1024 * 1024; // 20MB (absolute safety cap)
+function parsePositiveIntEnv(name) {
+    const raw = process.env[name];
+    if (!raw)
+        return undefined;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n <= 0)
+        return undefined;
+    return Math.floor(n);
+}
+function clamp(n, min, max) {
+    return Math.max(min, Math.min(max, n));
+}
 // =============================================================================
 // Registry Client
 // =============================================================================
 export class RegistryClient {
     registryUrl;
+    timeoutMs;
+    maxBytes;
     cache = { data: null, timestamp: 0 };
-    constructor(registryUrl = DEFAULT_REGISTRY_URL) {
-        this.registryUrl = registryUrl;
+    constructor(registryUrl, options = {}) {
+        const fromEnv = process.env.COGNITIVE_REGISTRY_URL;
+        const selected = (typeof registryUrl === 'string' && registryUrl.trim() ? registryUrl.trim() : undefined) ??
+            (typeof fromEnv === 'string' && fromEnv.trim() ? fromEnv.trim() : undefined) ??
+            DEFAULT_REGISTRY_URL;
+        this.registryUrl = selected;
+        const envTimeout = parsePositiveIntEnv('COGNITIVE_REGISTRY_TIMEOUT_MS');
+        const envMaxBytes = parsePositiveIntEnv('COGNITIVE_REGISTRY_MAX_BYTES');
+        const timeout = options.timeoutMs ?? envTimeout ?? DEFAULT_REGISTRY_FETCH_TIMEOUT_MS;
+        // keep timeouts bounded for predictable UX
+        this.timeoutMs = clamp(timeout, 1000, 120_000);
+        const maxBytes = options.maxBytes ?? envMaxBytes ?? DEFAULT_MAX_REGISTRY_BYTES;
+        // enforce an absolute upper bound to avoid accidental OOM on hostile endpoints
+        this.maxBytes = clamp(maxBytes, 1024, HARD_MAX_REGISTRY_BYTES);
     }
     async parseRegistryResponse(response) {
         const contentLengthHeader = response.headers?.get('content-length');
         if (contentLengthHeader) {
             const contentLength = Number(contentLengthHeader);
-            if (!Number.isNaN(contentLength) && contentLength > MAX_REGISTRY_BYTES) {
-                throw new Error(`Registry payload too large: ${contentLength} bytes (max ${MAX_REGISTRY_BYTES})`);
+            if (!Number.isNaN(contentLength) && contentLength > this.maxBytes) {
+                throw new Error(`Registry payload too large: ${contentLength} bytes (max ${this.maxBytes})`);
             }
         }
         if (response.body && typeof response.body.getReader === 'function') {
@@ -50,8 +84,8 @@ export class RegistryClient {
                         break;
                     if (value) {
                         totalBytes += value.byteLength;
-                        if (totalBytes > MAX_REGISTRY_BYTES) {
-                            throw new Error(`Registry payload too large: ${totalBytes} bytes (max ${MAX_REGISTRY_BYTES})`);
+                        if (totalBytes > this.maxBytes) {
+                            throw new Error(`Registry payload too large: ${totalBytes} bytes (max ${this.maxBytes})`);
                         }
                         buffer += decoder.decode(value, { stream: true });
                     }
@@ -71,8 +105,8 @@ export class RegistryClient {
         if (typeof response.text === 'function') {
             const text = await response.text();
             const byteLen = Buffer.byteLength(text, 'utf-8');
-            if (byteLen > MAX_REGISTRY_BYTES) {
-                throw new Error(`Registry payload too large: ${byteLen} bytes (max ${MAX_REGISTRY_BYTES})`);
+            if (byteLen > this.maxBytes) {
+                throw new Error(`Registry payload too large: ${byteLen} bytes (max ${this.maxBytes})`);
             }
             try {
                 return JSON.parse(text);
@@ -118,28 +152,48 @@ export class RegistryClient {
                 // Ignore cache read errors
             }
         }
-        // Fetch from network
-        const controller = new AbortController();
-        const timeout = setTimeout(() => controller.abort(), REGISTRY_FETCH_TIMEOUT_MS);
+        const fetchOnce = async (url) => {
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+            try {
+                const response = await fetch(url, {
+                    headers: { 'User-Agent': 'cognitive-runtime/2.2' },
+                    signal: controller.signal,
+                });
+                if (!response.ok) {
+                    // Allow callers to inspect status for fallback logic.
+                    const err = new Error(`Failed to fetch registry: ${response.status} ${response.statusText}`);
+                    err.status = response.status;
+                    throw err;
+                }
+                return await this.parseRegistryResponse(response);
+            }
+            catch (error) {
+                if (error instanceof Error && error.name === 'AbortError') {
+                    throw new Error(`Registry fetch timed out after ${this.timeoutMs}ms`);
+                }
+                throw error;
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+        };
         let data;
         try {
-            const response = await fetch(this.registryUrl, {
-                headers: { 'User-Agent': 'cognitive-runtime/2.2' },
-                signal: controller.signal,
-            });
-            if (!response.ok) {
-                throw new Error(`Failed to fetch registry: ${response.status} ${response.statusText}`);
-            }
-            data = await this.parseRegistryResponse(response);
+            data = await fetchOnce(this.registryUrl);
         }
-        catch (error) {
-            if (error instanceof Error && error.name === 'AbortError') {
-                throw new Error(`Registry fetch timed out after ${REGISTRY_FETCH_TIMEOUT_MS}ms`);
+        catch (e) {
+            // Harden the "latest" strategy: right after publishing a GitHub Release,
+            // the index asset may not be uploaded yet (short 404 window). In that case,
+            // fall back to the repo-tracked index to keep `cog search/add` usable.
+            const status = e?.status;
+            const isDefaultUrl = this.registryUrl === DEFAULT_REGISTRY_URL;
+            if (isDefaultUrl && status === 404) {
+                data = await fetchOnce(FALLBACK_REGISTRY_URL);
+            }
+            else {
+                throw e;
             }
-            throw error;
-        }
-        finally {
-            clearTimeout(timeout);
         }
         // Update cache
         this.cache = { data, timestamp: now };
@@ -172,7 +226,9 @@ export class RegistryClient {
                 version: v2.identity.version,
                 description: v2.metadata.description,
                 author: v2.metadata.author,
-                source: v2.distribution.source || v2.distribution.tarball || '',
+                source: v2.distribution.tarball,
+                tarball: v2.distribution.tarball,
+                checksum: v2.distribution.checksum,
                 keywords: v2.metadata.keywords || [],
                 tier: v2.metadata.tier,
                 namespace: v2.identity.namespace,
@@ -335,6 +391,7 @@ export class RegistryClient {
         if (source.startsWith('http://') || source.startsWith('https://')) {
             return {
                 url: source,
+                checksum: module.checksum,
                 isGitHub: false,
             };
         }

package/dist/registry/tar.d.ts

@@ -0,0 +1,8 @@
+export interface ExtractTarGzOptions {
+    maxFiles?: number;
+    maxTotalBytes?: number;
+    maxSingleFileBytes?: number;
+    maxTarBytes?: number;
+}
+export declare function extractTarGzFile(tarGzPath: string, destDir: string, options?: ExtractTarGzOptions): Promise<string[]>;
+export declare function extractTarGzBuffer(gzBuffer: Buffer, destDir: string, options?: ExtractTarGzOptions): Promise<string[]>;

package/dist/registry/tar.js

@@ -0,0 +1,353 @@
+import { mkdir } from 'node:fs/promises';
+import { createReadStream, createWriteStream } from 'node:fs';
+import { resolve, sep, dirname } from 'node:path';
+import { createGunzip } from 'node:zlib';
+import { Readable, Transform } from 'node:stream';
+import { finished } from 'node:stream/promises';
+function isPathWithinRoot(rootDir, targetPath) {
+    const root = resolve(rootDir);
+    const target = resolve(targetPath);
+    return target === root || target.startsWith(root + sep);
+}
+function parseOctal(buf) {
+    const raw = buf.toString('utf-8').replace(/\0/g, '').trim();
+    if (!raw)
+        return 0;
+    // Tar uses octal ASCII. Some implementations pad with spaces or NUL.
+    return parseInt(raw, 8);
+}
+function parseHeader(block) {
+    // End of archive: two consecutive zero blocks. Caller handles the second.
+    if (block.every((b) => b === 0)) {
+        return null;
+    }
+    const name = block.subarray(0, 100).toString('utf-8').replace(/\0/g, '');
+    const size = parseOctal(block.subarray(124, 136));
+    const typeflag = block.subarray(156, 157).toString('utf-8') || '\0';
+    const prefix = block.subarray(345, 500).toString('utf-8').replace(/\0/g, '');
+    return { name, size, typeflag, prefix };
+}
+function normalizeTarPath(path) {
+    const normalized = path.replace(/\\/g, '/');
+    if (!normalized || normalized.includes('\0')) {
+        throw new Error('Unsafe tar entry (empty or NUL)');
+    }
+    if (normalized.startsWith('/') || /^[a-zA-Z]:\//.test(normalized)) {
+        throw new Error(`Unsafe tar entry (absolute path): ${path}`);
+    }
+    const parts = normalized.split('/');
+    if (parts.includes('..')) {
+        throw new Error(`Unsafe tar entry (path traversal): ${path}`);
+    }
+    // Collapse `.` segments without allowing traversal.
+    const collapsed = parts.filter((p) => p !== '.' && p !== '').join('/');
+    if (!collapsed) {
+        throw new Error(`Unsafe tar entry (empty after normalize): ${path}`);
+    }
+    return collapsed;
+}
+function parsePaxAttributes(payload) {
+    // PAX format: "<len> <key>=<value>\n"
+    const text = payload.toString('utf-8');
+    const attrs = {};
+    let i = 0;
+    while (i < text.length) {
+        const space = text.indexOf(' ', i);
+        if (space === -1)
+            break;
+        const lenStr = text.slice(i, space);
+        const len = Number(lenStr);
+        if (!Number.isFinite(len) || len <= 0)
+            break;
+        const record = text.slice(i, i + len);
+        const eq = record.indexOf('=');
+        if (eq !== -1) {
+            const key = record.slice(record.indexOf(' ') + 1, eq).trim();
+            const value = record.slice(eq + 1).trimEnd();
+            // Strip the trailing newline if present.
+            attrs[key] = value.endsWith('\n') ? value.slice(0, -1) : value;
+        }
+        i += len;
+    }
+    return attrs;
+}
+function createByteLimitTransform(maxBytes) {
+    let seen = 0;
+    return new Transform({
+        transform(chunk, _encoding, callback) {
+            seen += chunk.length;
+            if (seen > maxBytes) {
+                callback(new Error(`Tar stream too large after decompression (max ${maxBytes} bytes)`));
+                return;
+            }
+            callback(null, chunk);
+        },
+    });
+}
+async function writeChunk(stream, chunk) {
+    if (!chunk.length)
+        return;
+    const ok = stream.write(chunk);
+    if (ok)
+        return;
+    await new Promise((resolveDrain, rejectDrain) => {
+        const onDrain = () => {
+            cleanup();
+            resolveDrain();
+        };
+        const onError = (err) => {
+            cleanup();
+            rejectDrain(err);
+        };
+        const cleanup = () => {
+            stream.off('drain', onDrain);
+            stream.off('error', onError);
+        };
+        stream.once('drain', onDrain);
+        stream.once('error', onError);
+    });
+}
+async function extractTarStream(tarStream, destDir, options = {}) {
+    const maxEntries = options.maxFiles ?? 5_000;
+    const maxTotalBytes = options.maxTotalBytes ?? 50 * 1024 * 1024; // 50MB extracted content cap
+    const maxSingleFileBytes = options.maxSingleFileBytes ?? 20 * 1024 * 1024; // 20MB per file cap
+    // Guard metadata record sizes so we don't buffer huge PAX/longname payloads into memory.
+    const MAX_META_BYTES = 1024 * 1024; // 1MB
+    let extractedBytes = 0;
+    let entriesSeen = 0;
+    const written = [];
+    let pendingPax = null;
+    let pendingLongName = null;
+    let buf = Buffer.alloc(0);
+    let pending = null;
+    let pendingPayload = Buffer.alloc(0); // only used for pax/longname/globalPax
+    let padRemaining = 0;
+    let ended = false;
+    const closePendingFile = async () => {
+        if (!pending || pending.kind !== 'file')
+            return;
+        const ws = pending.stream;
+        ws.end?.();
+        try {
+            await finished(ws);
+        }
+        catch {
+            // ignore (best-effort cleanup)
+        }
+    };
+    try {
+        for await (const chunk of tarStream) {
+            if (!chunk?.length)
+                continue;
+            buf = buf.length ? Buffer.concat([buf, chunk]) : Buffer.from(chunk);
+            while (true) {
+                if (ended)
+                    break;
+                // Drain padding to 512-byte boundary after each entry payload.
+                if (padRemaining > 0) {
+                    if (buf.length === 0)
+                        break;
+                    const take = Math.min(padRemaining, buf.length);
+                    buf = buf.subarray(take);
+                    padRemaining -= take;
+                    continue;
+                }
+                // If we are currently consuming an entry payload, continue that first.
+                if (pending) {
+                    if (pending.kind === 'file') {
+                        if (pending.remaining === 0) {
+                            // Zero-length file: create empty file and continue.
+                            await closePendingFile();
+                            written.push(pending.pathRel);
+                            pending = null;
+                            pendingPayload = Buffer.alloc(0);
+                            continue;
+                        }
+                        if (buf.length === 0)
+                            break;
+                        const take = Math.min(pending.remaining, buf.length);
+                        const slice = buf.subarray(0, take);
+                        await writeChunk(pending.stream, slice);
+                        pending.remaining -= take;
+                        buf = buf.subarray(take);
+                        if (pending.remaining > 0)
+                            continue;
+                        padRemaining = (512 - (pending.originalSize % 512)) % 512;
+                        extractedBytes += pending.originalSize;
+                        if (extractedBytes > maxTotalBytes) {
+                            throw new Error(`Tar extracted content too large (max ${maxTotalBytes} bytes)`);
+                        }
+                        await closePendingFile();
+                        written.push(pending.pathRel);
+                        pending = null;
+                        pendingPayload = Buffer.alloc(0);
+                        continue;
+                    }
+                    // Non-file payloads (PAX/longname/globalPax/skip).
+                    if (pending.remaining === 0) {
+                        padRemaining = (512 - (pending.originalSize % 512)) % 512;
+                        if (pending.kind === 'pax') {
+                            pendingPax = parsePaxAttributes(pendingPayload);
+                        }
+                        else if (pending.kind === 'longname') {
+                            const longName = pendingPayload.toString('utf-8').replace(/\0/g, '').trim();
+                            if (longName)
+                                pendingLongName = longName;
+                        }
+                        else if (pending.kind === 'globalPax') {
+                            // ignored
+                        }
+                        else if (pending.kind === 'skip') {
+                            // nothing
+                        }
+                        pending = null;
+                        pendingPayload = Buffer.alloc(0);
+                        continue;
+                    }
+                    if (buf.length === 0)
+                        break;
+                    const take = Math.min(pending.remaining, buf.length);
+                    const slice = buf.subarray(0, take);
+                    buf = buf.subarray(take);
+                    pending.remaining -= take;
+                    if (pending.kind !== 'skip') {
+                        pendingPayload = pendingPayload.length ? Buffer.concat([pendingPayload, slice]) : slice;
+                        if (pendingPayload.length > MAX_META_BYTES) {
+                            throw new Error(`Tar metadata entry too large (max ${MAX_META_BYTES} bytes)`);
+                        }
+                    }
+                    if (pending.remaining > 0)
+                        continue;
+                    padRemaining = (512 - (pending.originalSize % 512)) % 512;
+                    if (pending.kind === 'pax') {
+                        pendingPax = parsePaxAttributes(pendingPayload);
+                    }
+                    else if (pending.kind === 'longname') {
+                        const longName = pendingPayload.toString('utf-8').replace(/\0/g, '').trim();
+                        if (longName)
+                            pendingLongName = longName;
+                    }
+                    pending = null;
+                    pendingPayload = Buffer.alloc(0);
+                    continue;
+                }
+                // Need a header block.
+                if (buf.length < 512)
+                    break;
+                const headerBlock = buf.subarray(0, 512);
+                buf = buf.subarray(512);
+                const header = parseHeader(headerBlock);
+                if (!header) {
+                    ended = true;
+                    break;
+                }
+                entriesSeen += 1;
+                if (entriesSeen > maxEntries) {
+                    throw new Error(`Tar contains too many entries (max ${maxEntries})`);
+                }
+                let entryName = header.prefix ? `${header.prefix}/${header.name}` : header.name;
+                if (pendingLongName) {
+                    entryName = pendingLongName;
+                    pendingLongName = null;
+                }
+                if (pendingPax?.path) {
+                    entryName = pendingPax.path;
+                }
+                pendingPax = null;
+                const size = header.size;
+                // Reject symlinks/hardlinks/devices/etc. Only support files + dirs + metadata.
+                if (header.typeflag === '2' || header.typeflag === '1') {
+                    throw new Error(`Refusing to extract link entry: ${entryName}`);
+                }
+                if (header.typeflag === 'x') {
+                    if (size > MAX_META_BYTES) {
+                        throw new Error(`Tar metadata entry too large (max ${MAX_META_BYTES} bytes)`);
+                    }
+                    pending = { kind: 'pax', remaining: size, originalSize: size };
+                    padRemaining = 0;
+                    pendingPayload = Buffer.alloc(0);
+                    continue;
+                }
+                if (header.typeflag === 'g') {
+                    if (size > MAX_META_BYTES) {
+                        throw new Error(`Tar metadata entry too large (max ${MAX_META_BYTES} bytes)`);
+                    }
+                    pending = { kind: 'globalPax', remaining: size, originalSize: size };
+                    padRemaining = 0;
+                    pendingPayload = Buffer.alloc(0);
+                    continue;
+                }
+                if (header.typeflag === 'L') {
+                    if (size > MAX_META_BYTES) {
+                        throw new Error(`Tar metadata entry too large (max ${MAX_META_BYTES} bytes)`);
+                    }
+                    pending = { kind: 'longname', remaining: size, originalSize: size };
+                    padRemaining = 0;
+                    pendingPayload = Buffer.alloc(0);
+                    continue;
+                }
+                if (header.typeflag !== '0' && header.typeflag !== '\0' && header.typeflag !== '5') {
+                    throw new Error(`Unsupported tar entry type '${header.typeflag}' for ${entryName}`);
+                }
+                const rel = normalizeTarPath(entryName);
+                const outPath = resolve(destDir, rel);
+                if (!isPathWithinRoot(destDir, outPath)) {
+                    throw new Error(`Unsafe tar entry (outside dest): ${rel}`);
+                }
+                if (header.typeflag === '5') {
+                    // Directory entry.
+                    await mkdir(outPath, { recursive: true });
+                    // Directories may still have payload bytes (unusual); skip them safely.
+                    if (size > 0) {
+                        pending = { kind: 'skip', remaining: size, originalSize: size };
+                        padRemaining = 0;
+                        pendingPayload = Buffer.alloc(0);
+                    }
+                    continue;
+                }
+                // File entry.
+                if (size > maxSingleFileBytes) {
+                    throw new Error(`Tar entry too large: ${rel} (${size} bytes)`);
+                }
+                await mkdir(dirname(outPath), { recursive: true });
+                const ws = createWriteStream(outPath, { flags: 'w', mode: 0o644 });
+                // Track file size and padding. Keep originalSize so we can compute padding after consumption.
+                pending = {
+                    kind: 'file',
+                    pathRel: rel,
+                    outPath,
+                    remaining: size,
+                    originalSize: size,
+                    stream: ws,
+                };
+                padRemaining = 0;
+                pendingPayload = Buffer.alloc(0);
+            }
+        }
+    }
+    finally {
+        // Ensure pending file stream is closed on error.
+        await closePendingFile();
+    }
+    if (pending) {
+        throw new Error('Unexpected end of tar stream (truncated archive)');
+    }
+    if (padRemaining > 0) {
+        throw new Error('Unexpected end of tar stream (truncated padding)');
+    }
+    return written;
+}
+async function extractTarGzReadable(gzReadable, destDir, options = {}) {
+    const maxTarBytes = options.maxTarBytes ?? 100 * 1024 * 1024; // 100MB decompressed TAR cap
+    const gunzip = createGunzip();
+    const limited = gzReadable.pipe(gunzip).pipe(createByteLimitTransform(maxTarBytes));
+    return extractTarStream(limited, destDir, options);
+}
+export async function extractTarGzFile(tarGzPath, destDir, options = {}) {
+    const rs = createReadStream(tarGzPath);
+    return extractTarGzReadable(rs, destDir, options);
+}
+export async function extractTarGzBuffer(gzBuffer, destDir, options = {}) {
+    const rs = Readable.from([gzBuffer]);
+    return extractTarGzReadable(rs, destDir, options);
+}