@aiwg/cli 2026.8.3 → 2026.8.5

package/dist/src/serve/shared-host-scheduler.js

@@ -0,0 +1,260 @@
+/**
+ * Cross-orchestrator scheduling and admission for shared execution hosts.
+ *
+ * The store is the global serialization boundary. Multiple AIWG processes must
+ * use the same durable implementation; the in-memory store is for tests and
+ * single-process embedding only. Executor substrates report capacity and run
+ * admitted work, but do not own this policy.
+ *
+ * @implements #1566
+ */
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from 'node:fs';
+import { dirname } from 'node:path';
+export class InMemoryAdmissionStore {
+    snapshot = { revision: 0, records: {} };
+    transact(mutate) {
+        const draft = structuredClone(this.snapshot);
+        const result = mutate(draft);
+        draft.revision += 1;
+        this.snapshot = draft;
+        return result;
+    }
+    read() {
+        return structuredClone(this.snapshot);
+    }
+}
+/** Durable cross-process store. The lock is deliberately non-blocking: a
+ * concurrent writer receives a conflict and retries through its control loop. */
+export class FileAdmissionStore {
+    path;
+    constructor(path) {
+        this.path = path;
+    }
+    transact(mutate) {
+        mkdirSync(dirname(this.path), { recursive: true, mode: 0o700 });
+        const lockPath = `${this.path}.lock`;
+        let descriptor;
+        try {
+            descriptor = openSync(lockPath, 'wx', 0o600);
+        }
+        catch (error) {
+            if (error.code === 'EEXIST') {
+                throw new Error('shared-host admission store is busy; retry the request');
+            }
+            throw error;
+        }
+        try {
+            const snapshot = this.readUnsafe();
+            const result = mutate(snapshot);
+            snapshot.revision += 1;
+            const temporary = `${this.path}.${process.pid}.tmp`;
+            writeFileSync(temporary, `${JSON.stringify(snapshot, null, 2)}\n`, { mode: 0o600 });
+            renameSync(temporary, this.path);
+            return result;
+        }
+        finally {
+            closeSync(descriptor);
+            try {
+                unlinkSync(lockPath);
+            }
+            catch { /* best-effort lock cleanup */ }
+        }
+    }
+    read() {
+        return structuredClone(this.readUnsafe());
+    }
+    readUnsafe() {
+        if (!existsSync(this.path))
+            return { revision: 0, records: {} };
+        return JSON.parse(readFileSync(this.path, 'utf8'));
+    }
+}
+const terminalStates = new Set([
+    'denied', 'cancelled', 'timed-out', 'preempted',
+]);
+export class SharedHostScheduler {
+    store;
+    policy;
+    clock;
+    constructor(store, policy, clock = Date.now) {
+        this.store = store;
+        this.policy = policy;
+        this.clock = clock;
+        if (!Number.isInteger(policy.maxConcurrent) || policy.maxConcurrent < 1) {
+            throw new Error('maxConcurrent must be a positive integer');
+        }
+        if (policy.leaseTtlMs < 1 || policy.agingIntervalMs < 1) {
+            throw new Error('leaseTtlMs and agingIntervalMs must be positive');
+        }
+    }
+    submit(request) {
+        this.validateRequest(request);
+        return this.store.transact(snapshot => {
+            const existing = snapshot.records[request.requestId];
+            if (existing) {
+                if (!sameRequest(existing, request)) {
+                    throw new Error(`request '${request.requestId}' conflicts with an existing admission`);
+                }
+                return existing;
+            }
+            snapshot.records[request.requestId] = {
+                ...structuredClone(request),
+                state: 'queued',
+                revision: 1,
+                reason: 'awaiting shared-host capacity',
+            };
+            this.reconcile(snapshot);
+            return structuredClone(snapshot.records[request.requestId]);
+        });
+    }
+    reconcileNow() {
+        return this.store.transact(snapshot => {
+            this.reconcile(snapshot);
+            return structuredClone(snapshot);
+        });
+    }
+    renew(requestId) {
+        return this.store.transact(snapshot => {
+            const record = this.required(snapshot, requestId);
+            if (record.state !== 'admitted')
+                throw new Error(`cannot renew ${record.state} admission`);
+            record.leaseExpiresAt = new Date(this.clock() + this.policy.leaseTtlMs).toISOString();
+            record.revision += 1;
+            record.reason = 'lease renewed';
+            return structuredClone(record);
+        });
+    }
+    release(requestId) {
+        return this.store.transact(snapshot => {
+            const record = this.required(snapshot, requestId);
+            delete snapshot.records[requestId];
+            if (record.state === 'admitted')
+                this.reconcile(snapshot);
+            return structuredClone(snapshot);
+        });
+    }
+    cancel(requestId) {
+        return this.store.transact(snapshot => {
+            const record = this.required(snapshot, requestId);
+            if (terminalStates.has(record.state))
+                return structuredClone(record);
+            record.state = 'cancelled';
+            record.reason = 'cancelled by orchestrator';
+            record.finishedAt = new Date(this.clock()).toISOString();
+            record.revision += 1;
+            this.reconcile(snapshot);
+            return structuredClone(record);
+        });
+    }
+    snapshot() {
+        return this.store.read();
+    }
+    reconcile(snapshot) {
+        const now = this.clock();
+        for (const record of Object.values(snapshot.records)) {
+            if (record.state === 'admitted' && Date.parse(record.leaseExpiresAt ?? '') <= now) {
+                record.state = 'timed-out';
+                record.reason = 'admission lease expired; capacity recovered';
+                record.finishedAt = new Date(now).toISOString();
+                record.revision += 1;
+            }
+            else if (record.state === 'queued' && Date.parse(record.submittedAt) + record.queueTimeoutMs <= now) {
+                record.state = 'timed-out';
+                record.reason = 'queue deadline elapsed';
+                record.finishedAt = new Date(now).toISOString();
+                record.revision += 1;
+            }
+        }
+        let queued = Object.values(snapshot.records)
+            .filter(record => record.state === 'queued')
+            .sort((a, b) => this.compare(a, b, now));
+        for (const candidate of queued) {
+            if (!this.hasCapacity(snapshot, candidate)) {
+                if (this.policy.allowPreemption)
+                    this.tryPreempt(snapshot, candidate, now);
+            }
+            if (!this.hasCapacity(snapshot, candidate))
+                continue;
+            candidate.state = 'admitted';
+            candidate.reason = 'admitted by shared-host policy';
+            candidate.admittedAt = new Date(now).toISOString();
+            candidate.leaseExpiresAt = new Date(now + this.policy.leaseTtlMs).toISOString();
+            candidate.revision += 1;
+        }
+        queued = [];
+    }
+    tryPreempt(snapshot, candidate, now) {
+        const victims = Object.values(snapshot.records)
+            .filter(record => record.state === 'admitted' && record.preemptible === true)
+            .sort((a, b) => this.compare(b, a, now));
+        const victim = victims.find(record => {
+            if (this.effectivePriority(record, now) >= this.effectivePriority(candidate, now))
+                return false;
+            const priorState = record.state;
+            record.state = 'preempted';
+            const freesRequiredCapacity = this.hasCapacity(snapshot, candidate);
+            record.state = priorState;
+            return freesRequiredCapacity;
+        });
+        if (!victim)
+            return;
+        victim.state = 'preempted';
+        victim.reason = `preempted by higher-priority request '${candidate.requestId}'`;
+        victim.preemptedBy = candidate.requestId;
+        victim.finishedAt = new Date(now).toISOString();
+        victim.revision += 1;
+    }
+    hasCapacity(snapshot, candidate) {
+        const active = Object.values(snapshot.records).filter(record => record.state === 'admitted');
+        if (active.length >= this.policy.maxConcurrent)
+            return false;
+        if (!belowQuota(active, 'environment', candidate.environment, this.policy.environmentQuotas))
+            return false;
+        if (!belowQuota(active, 'provider', candidate.provider, this.policy.providerQuotas))
+            return false;
+        const runtimeQuotas = {
+            host: this.policy.defaultHostQuota ?? 1,
+            ...this.policy.runtimeQuotas,
+        };
+        return belowQuota(active, 'runtimeKind', candidate.runtimeKind, runtimeQuotas);
+    }
+    effectivePriority(record, now) {
+        const waited = Math.max(0, now - Date.parse(record.submittedAt));
+        return record.priority + Math.floor(waited / this.policy.agingIntervalMs);
+    }
+    compare(a, b, now) {
+        return this.effectivePriority(b, now) - this.effectivePriority(a, now)
+            || Date.parse(a.submittedAt) - Date.parse(b.submittedAt)
+            || a.requestId.localeCompare(b.requestId);
+    }
+    required(snapshot, requestId) {
+        const record = snapshot.records[requestId];
+        if (!record)
+            throw new Error(`unknown admission request '${requestId}'`);
+        return record;
+    }
+    validateRequest(request) {
+        if (!request.requestId || !request.orchestratorId || !request.environment || !request.provider) {
+            throw new Error('request identity, orchestrator, environment, and provider are required');
+        }
+        if (!Number.isFinite(request.priority))
+            throw new Error('priority must be finite');
+        if (!Number.isFinite(Date.parse(request.submittedAt)))
+            throw new Error('submittedAt must be a timestamp');
+        if (request.queueTimeoutMs < 1)
+            throw new Error('queueTimeoutMs must be positive');
+    }
+}
+function belowQuota(active, field, value, quotas) {
+    const quota = quotas?.[String(value)];
+    if (quota === undefined)
+        return true;
+    return active.filter(record => record[field] === value).length < quota;
+}
+function sameRequest(record, request) {
+    return record.orchestratorId === request.orchestratorId
+        && record.environment === request.environment
+        && record.provider === request.provider
+        && record.runtimeKind === request.runtimeKind;
+}
+//# sourceMappingURL=shared-host-scheduler.js.map

package/dist/src/storage/backends/fortemi.js

@@ -37,6 +37,58 @@
  * @issue #972
  */
 const DEFAULT_MCP_SERVER = 'fortemi';
+const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
+export function resolveMcpRequestHeaders(server, environment = process.env) {
+    const headers = { ...(server.headers ?? {}) };
+    for (const [header, envName] of Object.entries(server.headerEnv ?? {})) {
+        if (!ENV_NAME.test(envName)) {
+            throw new Error(`storage(fortemi): invalid environment variable reference "${envName}"`);
+        }
+        const value = environment[envName];
+        if (!value) {
+            throw new Error(`storage(fortemi): required credential environment variable "${envName}" is not set`);
+        }
+        headers[header] = header.toLowerCase() === 'authorization' ? `Bearer ${value}` : value;
+    }
+    return headers;
+}
+export function validateRemoteMcpUrl(raw) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        throw new Error(`storage(fortemi): invalid MCP server URL "${raw}"`);
+    }
+    const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname);
+    if (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) {
+        throw new Error('storage(fortemi): remote MCP URLs must use HTTPS; HTTP is allowed only for loopback development');
+    }
+    return url;
+}
+export function unwrapMcpToolResult(result) {
+    if (!result || typeof result !== 'object')
+        return result;
+    const envelope = result;
+    if (envelope.isError) {
+        const detail = envelope.content
+            ?.filter((item) => item.type === 'text' && typeof item.text === 'string')
+            .map((item) => item.text)
+            .join('; ');
+        throw new Error(`storage(fortemi): MCP tool failed${detail ? `: ${detail}` : ''}`);
+    }
+    if (envelope.structuredContent !== undefined)
+        return envelope.structuredContent;
+    const text = envelope.content?.find((item) => item.type === 'text' && typeof item.text === 'string')?.text;
+    if (text === undefined)
+        return result;
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return { content: text };
+    }
+}
 export class FortemiAdapter {
     subsystem;
     mcpServer;
@@ -194,30 +246,60 @@ export class FortemiAdapter {
  * Implemented as a lazy import so tests that inject a stub never load
  * the SDK or touch the registry.
  */
-export const createDefaultMcpClient = async (serverName) => {
+export const createDefaultMcpClient = async (serverName, registryOverride, environment = process.env) => {
     const { McpServerRegistry } = await import('../../mcp/registry.js');
-    const registry = new McpServerRegistry();
+    const registry = registryOverride ?? new McpServerRegistry();
     const server = await registry.get(serverName);
     if (!server) {
         throw new Error(`storage(fortemi): MCP server "${serverName}" is not registered. ` +
             `Add it via "aiwg mcp add ${serverName} --command <cmd>" before using the fortemi backend.`);
     }
-    if (server.type !== 'stdio') {
-        throw new Error(`storage(fortemi): only stdio MCP servers are supported (got "${server.type}" for "${serverName}")`);
-    }
-    // Lazy import the SDK so tests that inject a stub don't pay the cost
+    // Lazy imports keep unit tests that inject a stub isolated from transports.
     const { Client } = await import('@modelcontextprotocol/sdk/client/index.js');
-    const { StdioClientTransport } = await import('@modelcontextprotocol/sdk/client/stdio.js');
-    const transport = new StdioClientTransport({
-        command: server.command ?? '',
-        args: server.args ?? [],
-        env: server.env,
-    });
+    let transport;
+    if (server.type === 'stdio') {
+        const { StdioClientTransport } = await import('@modelcontextprotocol/sdk/client/stdio.js');
+        transport = new StdioClientTransport({
+            command: server.command ?? '',
+            args: server.args ?? [],
+            env: server.env,
+        });
+    }
+    else {
+        if (!server.url) {
+            throw new Error(`storage(fortemi): MCP server "${serverName}" has no URL`);
+        }
+        const url = validateRemoteMcpUrl(server.url);
+        const headers = resolveMcpRequestHeaders(server, environment);
+        if (server.type === 'http') {
+            const { StreamableHTTPClientTransport } = await import('@modelcontextprotocol/sdk/client/streamableHttp.js');
+            transport = new StreamableHTTPClientTransport(url, {
+                requestInit: { headers },
+            });
+        }
+        else if (server.type === 'sse') {
+            const { SSEClientTransport } = await import('@modelcontextprotocol/sdk/client/sse.js');
+            transport = new SSEClientTransport(url, {
+                requestInit: { headers },
+                eventSourceInit: {
+                    fetch: async (input, init) => {
+                        const merged = new Headers(init?.headers);
+                        for (const [name, value] of Object.entries(headers))
+                            merged.set(name, value);
+                        return fetch(input, { ...init, headers: merged });
+                    },
+                },
+            });
+        }
+        else {
+            throw new Error(`storage(fortemi): unsupported MCP transport "${String(server.type)}"`);
+        }
+    }
     const client = new Client({ name: 'aiwg-storage-fortemi-adapter', version: '1.0.0' }, { capabilities: {} });
     await client.connect(transport);
     return {
         async callTool(name, args) {
-            return client.callTool({ name, arguments: args });
+            return unwrapMcpToolResult(await client.callTool({ name, arguments: args }));
         },
         async close() {
             await client.close();

package/dist/src/storage/cli.js

@@ -7,6 +7,7 @@
  *   test <subsystem>   — round-trip read/write/list/delete through the
  *                        configured backend
  *   migrate <subsystem> — copy entries from one backend to another
+ *   import-corpus       — ingest the local research corpus through a storage backend
  *
  * @design @.aiwg/architecture/storage-design.md (§7)
  * @issue #934
@@ -16,13 +17,13 @@
 import { randomUUID } from 'crypto';
 import { existsSync } from 'fs';
 import { mkdir, readFile, writeFile, appendFile } from 'fs/promises';
-import { dirname, join, resolve as resolvePath } from 'path';
+import { dirname, extname, join, resolve as resolvePath } from 'path';
+import { parseFrontmatter } from '../artifacts/index-builder.js';
 import { BACKEND_TYPES, FilesystemAdapter, ObsidianAdapter, LogseqAdapter, FortemiAdapter, SUBSYSTEM_KEYS, getLoadedConfig, initStorage, resolveStorage, resolveSubsystemRoot, storageConfigPath, } from './index.js';
 import { projectAiwgPath, resolveProjectAiwgDir } from '../config/project-artifacts.js';
-export async function main(args) {
+export async function main(args, projectRoot = process.cwd()) {
     const subcommand = args[0];
     const subArgs = args.slice(1);
-    const projectRoot = process.cwd();
     switch (subcommand) {
         case 'show':
             await handleShow(projectRoot);
@@ -36,6 +37,9 @@ export async function main(args) {
         case 'migrate':
             await handleMigrate(projectRoot, subArgs);
             break;
+        case 'import-corpus':
+            await handleImportCorpus(projectRoot, subArgs);
+            break;
         default:
             printUsage();
             if (subcommand) {
@@ -166,7 +170,7 @@ async function handleMigrate(projectRoot, args) {
     }
     if (source.init)
         await source.init();
-    if (destination.init)
+    if (!opts.dryRun && destination.init)
         await destination.init();
     console.log(`storage migrate (${opts.dryRun ? 'DRY RUN' : 'live'})`);
     console.log(`  subsystem: ${opts.subsystem}`);
@@ -184,7 +188,13 @@ async function handleMigrate(projectRoot, args) {
     let copied = 0;
     let skipped = 0;
     let errored = 0;
+    let unsupported = 0;
     for (const entry of entries) {
+        if (opts.textOnly && !isTextCorpusEntry(entry.path)) {
+            unsupported++;
+            console.log(`  · ${entry.path} (non-text attachment skipped)`);
+            continue;
+        }
         if (completed.has(entry.path)) {
             skipped++;
             console.log(`  ✓ ${entry.path} (already migrated, skipped)`);
@@ -202,7 +212,7 @@ async function handleMigrate(projectRoot, args) {
                 console.log(`  ✗ ${entry.path} (read returned null)`);
                 continue;
             }
-            await destination.write(entry.path, content);
+            await destination.write(entry.path, content, migrationMetadata(entry.path, content));
             await recordCompletion(migrationLogPath, entry.path);
             copied++;
             console.log(`  ✓ ${entry.path}`);
@@ -217,7 +227,8 @@ async function handleMigrate(projectRoot, args) {
     if (destination.close)
         await destination.close();
     console.log('');
-    console.log(`Summary: copied=${copied} skipped=${skipped} errored=${errored} total=${entries.length}`);
+    console.log(`Summary: copied=${copied} skipped=${skipped} unsupported=${unsupported} ` +
+        `errored=${errored} total=${entries.length}`);
     if (!opts.dryRun) {
         console.log(`Migration log: ${migrationLogPath}`);
     }
@@ -232,6 +243,7 @@ function parseMigrateArgs(args) {
     let fromFolder;
     let toFolder;
     let dryRun = false;
+    let textOnly = false;
     for (let i = 0; i < args.length; i++) {
         const a = args[i];
         if (a === '--from')
@@ -244,6 +256,8 @@ function parseMigrateArgs(args) {
             toFolder = args[++i];
         else if (a === '--dry-run')
             dryRun = true;
+        else if (a === '--text-only')
+            textOnly = true;
         else if (!a.startsWith('--') && !subsystem)
             subsystem = a;
         else
@@ -261,8 +275,73 @@ function parseMigrateArgs(args) {
         from: { ...parseSpec(from), ...(fromFolder ? { folder: fromFolder } : {}) },
         to: { ...parseSpec(to), ...(toFolder ? { folder: toFolder } : {}) },
         dryRun,
+        textOnly,
     };
 }
+const TEXT_CORPUS_EXTENSIONS = new Set([
+    '.bib',
+    '.csv',
+    '.htm',
+    '.html',
+    '.json',
+    '.md',
+    '.ris',
+    '.txt',
+    '.xml',
+    '.yaml',
+    '.yml',
+]);
+export function isTextCorpusEntry(entryPath) {
+    return TEXT_CORPUS_EXTENSIONS.has(extname(entryPath).toLowerCase());
+}
+function migrationMetadata(entryPath, content) {
+    const extension = extname(entryPath).toLowerCase();
+    const contentType = extension === '.md' ? 'text/markdown' : 'text/plain';
+    if (extension !== '.md')
+        return { contentType };
+    return { contentType, frontmatter: parseFrontmatter(content).data };
+}
+async function handleImportCorpus(projectRoot, args) {
+    let server = 'fortemi';
+    let destination;
+    let serverSelected = false;
+    let dryRun = false;
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === '--server') {
+            server = args[++index] ?? '';
+            serverSelected = true;
+            if (!server)
+                throw new Error('storage import-corpus: --server requires a name');
+        }
+        else if (arg === '--to') {
+            destination = args[++index] ?? '';
+            if (!destination)
+                throw new Error('storage import-corpus: --to requires a backend spec');
+        }
+        else if (arg === '--dry-run') {
+            dryRun = true;
+        }
+        else {
+            throw new Error(`Unknown import-corpus flag: ${arg}`);
+        }
+    }
+    if (serverSelected && destination) {
+        throw new Error('storage import-corpus: use either --server or --to, not both');
+    }
+    await initStorage(projectRoot);
+    const config = await getLoadedConfig(projectRoot);
+    const sourceRoot = resolveSubsystemRoot('research', projectRoot, config);
+    await handleMigrate(projectRoot, [
+        'research',
+        '--from',
+        `fs:${sourceRoot}`,
+        '--to',
+        destination ?? `fortemi:${server}`,
+        '--text-only',
+        ...(dryRun ? ['--dry-run'] : []),
+    ]);
+}
 function parseSpec(raw) {
     const idx = raw.indexOf(':');
     if (idx === -1) {
@@ -449,11 +528,16 @@ Subcommands:
   list-backends [--json]        Inventory of compiled-in adapters; --json emits structured output including tracking_issue URL for stubs
   test <subsystem>              Round-trip read/write/list/delete through the configured backend
   migrate <subsystem>           Copy entries from one backend to another (#955)
+  import-corpus                 Ingest local research text through a storage backend (#1508)
+    --to <type>:<location>      Provider-neutral destination backend (default: fortemi:fortemi)
+    --server <name>             MCP registry server name (default: fortemi)
+    --dry-run                   Preview corpus selection without connecting
     --from <type>:<location>    Source spec (fs:./dir, obsidian:~/vault, logseq:./graph, fortemi:server)
     --to <type>:<location>      Destination spec (same format)
     --from-folder <folder>      Optional Obsidian subfolder for source
     --to-folder <folder>        Optional Obsidian subfolder for destination
     --dry-run                   Preview operations without writing
+    --text-only                 Skip non-text attachments
 
 Subsystems: ${SUBSYSTEM_KEYS.join(', ')}
 
@@ -463,6 +547,9 @@ Examples:
   aiwg storage test activity_log
   aiwg storage migrate memory --from fs:.aiwg/memory --to obsidian:~/vault --to-folder AIWG/memory --dry-run
   aiwg storage migrate kb --from fs:.aiwg/kb --to fortemi:fortemi
+  aiwg storage import-corpus --dry-run
+  aiwg storage import-corpus --to obsidian:~/vault --dry-run
+  aiwg storage import-corpus --server fortemi-enterprise
 
 See @.aiwg/architecture/storage-design.md for the design.`);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiwg/cli",
-  "version": "2026.8.3",
+  "version": "2026.8.5",
   "description": "Lightweight AIWG CLI for signed, versioned web-backed resources.",
   "type": "module",
   "license": "MIT",