cognitive-modules-cli 2.2.5 → 2.2.8

package/dist/commands/add.js

@@ -9,13 +9,16 @@
  *   cog add ziel-io/cognitive-modules -m code-simplifier
  *   cog add https://github.com/org/repo --module name --tag v1.0.0
  */
-import { createWriteStream, existsSync, mkdirSync, rmSync, readdirSync, statSync, copyFileSync } from 'node:fs';
+import { createWriteStream, existsSync, mkdirSync, rmSync, readdirSync, statSync, copyFileSync, lstatSync } from 'node:fs';
 import { writeFile, readFile, mkdir } from 'node:fs/promises';
-import { pipeline } from 'node:stream/promises';
+import { pipeline, finished } from 'node:stream/promises';
 import { Readable } from 'node:stream';
 import { join, basename, dirname, resolve, sep, isAbsolute } from 'node:path';
 import { homedir, tmpdir } from 'node:os';
+import { createHash } from 'node:crypto';
 import { RegistryClient } from '../registry/client.js';
+import { extractTarGzFile } from '../registry/tar.js';
+import { PROVENANCE_SPEC, computeModuleIntegrity, writeModuleProvenance } from '../provenance.js';
 // Module storage paths
 const USER_MODULES_DIR = join(homedir(), '.cognitive', 'modules');
 const INSTALLED_MANIFEST = join(homedir(), '.cognitive', 'installed.json');
@@ -153,7 +156,11 @@ function copyDir(src, dest) {
     for (const entry of readdirSync(src)) {
         const srcPath = join(src, entry);
         const destPath = join(dest, entry);
-        if (statSync(srcPath).isDirectory()) {
+        const st = lstatSync(srcPath);
+        if (st.isSymbolicLink()) {
+            throw new Error(`Refusing to install module containing symlink: ${srcPath}`);
+        }
+        if (st.isDirectory()) {
             copyDir(srcPath, destPath);
         }
         else {
@@ -161,6 +168,74 @@ function copyDir(src, dest) {
         }
     }
 }
+async function downloadTarballWithSha256(url, outPath, maxBytes) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 10_000);
+    const hash = createHash('sha256');
+    let received = 0;
+    const fileStream = createWriteStream(outPath);
+    try {
+        const response = await fetch(url, {
+            headers: { 'User-Agent': 'cognitive-runtime/2.2' },
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to download tarball: ${response.status} ${response.statusText}`);
+        }
+        const contentLengthHeader = response.headers?.get?.('content-length');
+        if (contentLengthHeader) {
+            const contentLength = Number(contentLengthHeader);
+            if (!Number.isNaN(contentLength) && contentLength > maxBytes) {
+                throw new Error(`Tarball too large: ${contentLength} bytes (max ${maxBytes})`);
+            }
+        }
+        if (!response.body) {
+            throw new Error('Tarball response has no body');
+        }
+        const reader = response.body.getReader?.();
+        if (!reader) {
+            // Fallback: stream pipeline without incremental hash.
+            await pipeline(Readable.fromWeb(response.body), fileStream);
+            const content = await readFile(outPath);
+            return createHash('sha256').update(content).digest('hex');
+        }
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (value) {
+                received += value.byteLength;
+                if (received > maxBytes) {
+                    controller.abort();
+                    throw new Error(`Tarball too large: ${received} bytes (max ${maxBytes})`);
+                }
+                const chunk = Buffer.from(value);
+                hash.update(chunk);
+                if (!fileStream.write(chunk)) {
+                    await new Promise((resolve) => fileStream.once('drain', () => resolve()));
+                }
+            }
+        }
+        fileStream.end();
+        await finished(fileStream);
+        return hash.digest('hex');
+    }
+    catch (error) {
+        try {
+            fileStream.destroy();
+        }
+        catch {
+            // ignore
+        }
+        if (error instanceof Error && error.name === 'AbortError') {
+            throw new Error('Tarball download timed out after 10000ms');
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
 /**
  * Get module version from module.yaml or MODULE.md
  */
@@ -266,8 +341,13 @@ function parseModuleSpec(spec) {
 export async function addFromRegistry(moduleSpec, ctx, options = {}) {
     const { name: moduleName, version: requestedVersion } = parseModuleSpec(moduleSpec);
     const { name: customName, registry: registryUrl } = options;
+    const policy = ctx.policy;
+    let tempDir;
     try {
-        const client = new RegistryClient(registryUrl);
+        const client = new RegistryClient(registryUrl, {
+            timeoutMs: ctx.registryTimeoutMs,
+            maxBytes: ctx.registryMaxBytes,
+        });
         const moduleInfo = await client.getModule(moduleName);
         if (!moduleInfo) {
             return {
@@ -282,6 +362,14 @@ export async function addFromRegistry(moduleSpec, ctx, options = {}) {
         // Get download info
         const downloadInfo = await client.getDownloadUrl(moduleName);
         if (downloadInfo.isGitHub && downloadInfo.githubInfo) {
+            if (policy?.profile === 'certified') {
+                return {
+                    success: false,
+                    error: `Certified profile requires registry tarball provenance.\n` +
+                        `Registry entry for '${moduleName}' resolves to a GitHub source, which is not allowed in --profile certified.\n` +
+                        `Use a tarball-based registry entry (distribution.tarball + checksum), or run with --profile strict/default.`,
+                };
+            }
             const { org, repo, path, ref } = downloadInfo.githubInfo;
             // Use addFromGitHub() directly (not add() to avoid recursion)
             const result = await addFromGitHub(`${org}/${repo}`, ctx, {
@@ -325,10 +413,112 @@ export async function addFromRegistry(moduleSpec, ctx, options = {}) {
             }
             return result;
         }
-        // For tarball sources, download directly (future implementation)
+        // Tarball sources: require checksum, verify, and extract safely.
+        if (!downloadInfo.url.startsWith('http')) {
+            return { success: false, error: `Unsupported registry download URL: ${downloadInfo.url}` };
+        }
+        if (!downloadInfo.checksum) {
+            return {
+                success: false,
+                error: `Registry tarball missing checksum (required for safe install): ${moduleName}`,
+            };
+        }
+        tempDir = join(tmpdir(), `cog-reg-${Date.now()}`);
+        mkdirSync(tempDir, { recursive: true });
+        const tarPath = join(tempDir, 'module.tar.gz');
+        const MAX_TARBALL_BYTES = 20 * 1024 * 1024; // 20MB
+        const actualSha256 = await downloadTarballWithSha256(downloadInfo.url, tarPath, MAX_TARBALL_BYTES);
+        const expected = downloadInfo.checksum;
+        const checksumMatch = expected.match(/^sha256:([a-f0-9]{64})$/);
+        if (!checksumMatch) {
+            throw new Error(`Unsupported checksum format (expected sha256:<64hex>): ${expected}`);
+        }
+        const expectedHash = checksumMatch[1];
+        if (actualSha256 !== expectedHash) {
+            throw new Error(`Checksum mismatch for ${moduleName}: expected ${expectedHash}, got ${actualSha256}`);
+        }
+        const extractedRoot = join(tempDir, 'pkg');
+        mkdirSync(extractedRoot, { recursive: true });
+        await extractTarGzFile(tarPath, extractedRoot, {
+            maxFiles: 5_000,
+            maxTotalBytes: 50 * 1024 * 1024,
+            maxSingleFileBytes: 20 * 1024 * 1024,
+            maxTarBytes: 100 * 1024 * 1024,
+        });
+        // Find module directory inside extractedRoot.
+        const rootNames = readdirSync(extractedRoot).filter((e) => e !== '__MACOSX' && e !== '.DS_Store');
+        const rootPaths = rootNames.map((e) => join(extractedRoot, e)).filter((p) => existsSync(p));
+        const rootDirs = rootPaths.filter((p) => statSync(p).isDirectory());
+        const rootFiles = rootPaths.filter((p) => !statSync(p).isDirectory());
+        if (rootDirs.length === 0) {
+            throw new Error('Tarball extraction produced no root directory');
+        }
+        if (rootDirs.length !== 1 || rootFiles.length > 0) {
+            throw new Error(`Tarball must contain exactly one module root directory and no other top-level entries. ` +
+                `dirs=${rootDirs.map((p) => basename(p)).join(',') || '(none)'} files=${rootFiles.map((p) => basename(p)).join(',') || '(none)'}`);
+        }
+        // Strict mode: require root dir itself to be a valid module.
+        const sourcePath = rootDirs[0];
+        if (!isValidModule(sourcePath)) {
+            throw new Error('Root directory in tarball is not a valid module');
+        }
+        const installName = (customName || moduleName);
+        const safeInstallName = assertSafeModuleName(installName);
+        const targetPath = resolveModuleTarget(safeInstallName);
+        if (existsSync(targetPath)) {
+            rmSync(targetPath, { recursive: true, force: true });
+        }
+        await mkdir(USER_MODULES_DIR, { recursive: true });
+        copyDir(sourcePath, targetPath);
+        const version = await getModuleVersion(sourcePath);
+        // Write provenance + integrity (enables certified profile gating).
+        try {
+            const integrity = await computeModuleIntegrity(targetPath);
+            await writeModuleProvenance(targetPath, {
+                spec: PROVENANCE_SPEC,
+                createdAt: new Date().toISOString(),
+                source: {
+                    type: 'registry',
+                    registryUrl: registryUrl ?? null,
+                    moduleName,
+                    requestedVersion: requestedVersion ?? null,
+                    resolvedVersion: version ?? null,
+                    tarballUrl: downloadInfo.url,
+                    checksum: downloadInfo.checksum,
+                    sha256: actualSha256,
+                    quality: {
+                        // moduleInfo is typed loosely (v1/v2); best-effort extract for v2.
+                        verified: moduleInfo?.quality?.verified,
+                        conformance_level: moduleInfo?.quality?.conformance_level,
+                        spec_version: moduleInfo?.identity?.spec_version,
+                    },
+                },
+                integrity,
+            });
+        }
+        catch (e) {
+            // If provenance fails, keep install but warn loudly (non-certified users can still run).
+            console.error(`Warning: failed to write provenance for ${safeInstallName}: ${e.message}`);
+        }
+        await recordInstall(safeInstallName, {
+            source: downloadInfo.url,
+            githubUrl: downloadInfo.url,
+            version,
+            installedAt: targetPath,
+            installedTime: new Date().toISOString(),
+            registryModule: moduleName,
+            registryUrl,
+        });
         return {
-            success: false,
-            error: `Tarball downloads not yet supported. Source: ${moduleInfo.source}`,
+            success: true,
+            data: {
+                message: `Added: ${safeInstallName}${version ? ` v${version}` : ''} (registry tarball)`,
+                name: safeInstallName,
+                version,
+                location: targetPath,
+                source: 'registry',
+                registryModule: moduleName,
+            },
         };
     }
     catch (error) {
@@ -337,6 +527,16 @@ export async function addFromRegistry(moduleSpec, ctx, options = {}) {
             error: error instanceof Error ? error.message : String(error),
         };
     }
+    finally {
+        if (tempDir) {
+            try {
+                rmSync(tempDir, { recursive: true, force: true });
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
 }
 /**
  * Add a module from GitHub
@@ -344,6 +544,14 @@ export async function addFromRegistry(moduleSpec, ctx, options = {}) {
 export async function addFromGitHub(url, ctx, options = {}) {
     const { org, repo, fullUrl } = parseGitHubUrl(url);
     const { module: modulePath, name, branch = 'main', tag } = options;
+    const policy = ctx.policy;
+    if (policy?.profile === 'certified') {
+        return {
+            success: false,
+            error: `Certified profile requires registry tarball provenance; GitHub installs are not allowed.\n` +
+                `Use 'cog add <module>' against a tarball-based registry entry, or run with --profile strict/default.`,
+        };
+    }
     // Determine ref (tag takes priority)
     const ref = tag || branch;
     const isTag = !!tag;
@@ -388,6 +596,19 @@ export async function addFromGitHub(url, ctx, options = {}) {
             installedAt: targetPath,
             installedTime: new Date().toISOString(),
         });
+        // Best-effort provenance for non-certified installs (useful for audit/debug).
+        try {
+            const integrity = await computeModuleIntegrity(targetPath);
+            await writeModuleProvenance(targetPath, {
+                spec: PROVENANCE_SPEC,
+                createdAt: new Date().toISOString(),
+                source: { type: 'github', repoUrl: fullUrl, ref, modulePath: modulePath ?? null },
+                integrity,
+            });
+        }
+        catch {
+            // ignore
+        }
         // Cleanup temp directory
         const tempDir = dirname(repoRoot);
         if (tempDir && tempDir !== '/' && tempDir !== '.' && tempDir !== USER_MODULES_DIR) {
@@ -427,6 +648,10 @@ export async function addFromGitHub(url, ctx, options = {}) {
  * Add a module from GitHub or Registry (auto-detect source)
  */
 export async function add(source, ctx, options = {}) {
+    // Allow a global registry override via ctx.registryUrl unless explicitly set.
+    if (!options.registry && ctx.registryUrl) {
+        options = { ...options, registry: ctx.registryUrl };
+    }
     // Determine source type
     if (isGitHubSource(source)) {
         return addFromGitHub(source, ctx, options);

package/dist/commands/compose.d.ts

@@ -13,6 +13,8 @@ export interface ComposeOptions {
     args?: string;
     /** JSON input data */
     input?: string;
+    /** Disable validation */
+    noValidate?: boolean;
     /** Maximum composition depth */
     maxDepth?: number;
     /** Timeout in milliseconds */

package/dist/commands/compose.js

@@ -9,6 +9,7 @@
  */
 import { findModule, getDefaultSearchPaths, executeComposition } from '../modules/index.js';
 import { ErrorCodes, attachContext, makeErrorEnvelope } from '../errors/index.js';
+import { writeAuditRecord } from '../audit.js';
 function looksLikeCode(str) {
     const codeIndicators = [
         /^(def|function|class|const|let|var|import|export|public|private)\s/,
@@ -34,7 +35,19 @@ export async function compose(moduleName, ctx, options = {}) {
             data: errorEnvelope,
         };
     }
+    if (ctx.policy?.requireV22) {
+        if (module.formatVersion !== 'v2.2') {
+            const errorEnvelope = attachContext(makeErrorEnvelope({
+                code: ErrorCodes.INVALID_INPUT,
+                message: `Certified profile requires v2.2 modules; got: ${module.formatVersion ?? 'unknown'} (${module.format})`,
+                suggestion: "Migrate the module to v2.2, or run with `--profile strict` / `--profile default`",
+            }), { module: moduleName, provider: ctx.provider.name });
+            return { success: false, error: errorEnvelope.error.message, data: errorEnvelope };
+        }
+    }
     try {
+        const policy = ctx.policy;
+        const startedAt = Date.now();
         // Parse input if provided as JSON
         let inputData = {};
         if (options.input) {
@@ -67,7 +80,31 @@ export async function compose(moduleName, ctx, options = {}) {
         const result = await executeComposition(moduleName, inputData, ctx.provider, {
             cwd: ctx.cwd,
             maxDepth: options.maxDepth,
-            timeoutMs: options.timeout
+            timeoutMs: options.timeout,
+            policy,
+            validateInput: (() => {
+                if (options.noValidate)
+                    return false;
+                if (!policy)
+                    return true;
+                if (policy.validate === 'off')
+                    return false;
+                if (policy.validate === 'on')
+                    return true;
+                return policy.profile !== 'core';
+            })(),
+            validateOutput: (() => {
+                if (options.noValidate)
+                    return false;
+                if (!policy)
+                    return true;
+                if (policy.validate === 'off')
+                    return false;
+                if (policy.validate === 'on')
+                    return true;
+                return policy.profile !== 'core';
+            })(),
+            enableRepair: policy?.enableRepair ?? true,
         });
         if (options.verbose) {
             console.error('--- Composition Trace ---');
@@ -98,6 +135,28 @@ export async function compose(moduleName, ctx, options = {}) {
                 data: errorEnvelope,
             };
         }
+        if (policy?.audit) {
+            const rec = await writeAuditRecord({
+                ts: new Date().toISOString(),
+                kind: 'compose',
+                policy,
+                provider: ctx.provider.name,
+                module: { name: module.name, version: module.version, location: module.location, formatVersion: module.formatVersion },
+                input: inputData,
+                result: {
+                    ok: result.ok,
+                    result: result.result,
+                    moduleResults: result.moduleResults,
+                    trace: result.trace,
+                    totalTimeMs: result.totalTimeMs,
+                    error: result.error,
+                },
+                notes: [`duration_ms=${Date.now() - startedAt}`],
+            });
+            if (rec) {
+                console.error(`Audit: ${rec.path}`);
+            }
+        }
         // Return result
         if (options.trace) {
             // Include full result with trace

package/dist/commands/core.d.ts

@@ -0,0 +1,31 @@
+/**
+ * cog core - Minimal "one-file" Core workflow
+ *
+ * Goals:
+ * - A single Markdown file can be a runnable module (optional frontmatter + prompt body)
+ * - Runtime generates loose schemas and always returns a v2.2 envelope on execution
+ * - No registry / conformance / certification required to get started
+ */
+import type { CommandContext, CommandResult } from '../types.js';
+export interface CoreOptions {
+    args?: string;
+    input?: string;
+    noValidate?: boolean;
+    pretty?: boolean;
+    verbose?: boolean;
+    stream?: boolean;
+    dryRun?: boolean;
+    stdin?: boolean;
+    force?: boolean;
+}
+export declare function coreNew(filePath: string, options?: {
+    dryRun?: boolean;
+}): Promise<CommandResult>;
+export declare function coreSchema(filePath: string): Promise<CommandResult>;
+export declare function corePromote(filePath: string, outDir?: string, options?: {
+    dryRun?: boolean;
+    force?: boolean;
+}): Promise<CommandResult>;
+export declare function coreRunText(markdownOrPrompt: string, ctx: CommandContext, options?: CoreOptions): Promise<CommandResult>;
+export declare function coreRun(filePath: string, ctx: CommandContext, options?: CoreOptions): Promise<CommandResult>;
+export declare function core(subcommand: string | undefined, target: string | undefined, ctx: CommandContext, options?: CoreOptions, rest?: string[]): Promise<CommandResult>;