@aiwg/cli 2026.10.1 → 2026.10.4

package/dist/src/sessions/readers.js

@@ -1,15 +1,82 @@
 import { createHash } from 'node:crypto';
-import { createReadStream } from 'node:fs';
 import { open, readFile } from 'node:fs/promises';
-import { createInterface } from 'node:readline';
-import { SessionContractError } from './contracts.js';
+import { assertSessionResourceLimit, schemaIssueDetail, SessionContractError } from './contracts.js';
+import { assertImportByteLimit, DEFAULT_IMPORT_BYTE_LIMITS } from './import-limits.js';
 import { authorizeSourceFile } from './policy.js';
 export const DEFAULT_READER_LIMITS = Object.freeze({
     maxRecords: 1_000_000,
-    maxRecordBytes: 8 * 1024 * 1024,
-    maxTotalBytes: 1024 * 1024 * 1024,
+    ...DEFAULT_IMPORT_BYTE_LIMITS,
     maxNestingDepth: 64,
 });
+const JSONL_READ_CHUNK_BYTES = 1024 * 1024;
+export async function* streamJsonLines(path, options = {}) {
+    const startOffset = options.start ?? 0;
+    if (!Number.isSafeInteger(startOffset) || startOffset < 0)
+        throw new RangeError('start must be a nonnegative safe integer');
+    if (options.end !== undefined && options.end !== Infinity
+        && (!Number.isSafeInteger(options.end) || options.end < 0))
+        throw new RangeError('end must be a nonnegative safe integer or Infinity');
+    if (options.end !== undefined && options.end < startOffset)
+        throw new RangeError('end must not precede start');
+    // Reuse raw storage across the full source instead of allocating a new MiB
+    // buffer per read. Only incomplete line fragments outlive the current read.
+    const storage = Buffer.allocUnsafe(JSONL_READ_CHUNK_BYTES);
+    const input = await open(path, 'r');
+    let position = startOffset;
+    let parts = [];
+    let length = 0;
+    let total = 0;
+    try {
+        while (options.end === undefined || position <= options.end) {
+            const wanted = options.end === undefined ? storage.length
+                : Math.min(storage.length, options.end - position + 1);
+            const { bytesRead } = await input.read(storage, 0, wanted, position);
+            if (bytesRead === 0)
+                break;
+            position += bytesRead;
+            const buffer = storage.subarray(0, bytesRead);
+            let start = 0;
+            while (start < buffer.length) {
+                const newline = buffer.indexOf(10, start);
+                const end = newline === -1 ? buffer.length : newline + 1;
+                const segment = buffer.subarray(start, end);
+                if (options.limits) {
+                    const recordBytes = length + segment.length;
+                    const totalBytes = (options.start ?? 0) + total + segment.length;
+                    // For an incomplete segment, report the first byte known to exceed
+                    // the limit instead of making diagnostics depend on read-buffer size.
+                    assertImportByteLimit('maxRecordBytes', options.limits, newline === -1 && recordBytes > options.limits.maxRecordBytes
+                        ? options.limits.maxRecordBytes + 1 : recordBytes);
+                    assertImportByteLimit('maxTotalBytes', options.limits, newline === -1 && options.limits.maxTotalBytes !== 0 && totalBytes > options.limits.maxTotalBytes
+                        ? options.limits.maxTotalBytes + 1 : totalBytes);
+                }
+                // Bound the raw bytes before copying or decoding a record. Only LF
+                // separates records; CRLF and Unicode separators remain JSON input.
+                parts.push(segment);
+                length += segment.length;
+                total += segment.length;
+                start = end;
+                if (newline !== -1) {
+                    const line = (parts.length === 1 ? parts[0] : Buffer.concat(parts, length)).toString('utf8');
+                    const byteLength = length;
+                    parts = [];
+                    length = 0;
+                    yield { line, byteLength };
+                }
+            }
+            // Earlier fragments already own their bytes. The last fragment is the
+            // only view into storage, which the next read will overwrite.
+            if (parts.length > 0)
+                parts[parts.length - 1] = Buffer.from(parts[parts.length - 1]);
+        }
+        if (length > 0) {
+            yield { line: (parts.length === 1 ? parts[0] : Buffer.concat(parts, length)).toString('utf8'), byteLength: length };
+        }
+    }
+    finally {
+        await input.close();
+    }
+}
 export async function readBoundedJsonLines(authorization, options) {
     const stream = await streamBoundedJsonLines(authorization, options);
     const records = [];
@@ -28,50 +95,36 @@ export async function streamBoundedJsonLines(authorization, options) {
     const limits = { ...DEFAULT_READER_LIMITS, ...options.limits };
     const start = parseCursor(options.cursor);
     if (start > allowed.size)
-        throw new SessionContractError('SCHEMA_DRIFT', 'reader cursor is beyond source size');
+        throw new SessionContractError('SCHEMA_DRIFT', 'reader cursor is beyond source size', schemaIssueDetail('cursor'));
     let offset = start;
     let total = 0;
     let count = 0;
     let incompleteTail = false;
     const iterate = async function* () {
-        const input = createReadStream(allowed.canonicalPath, { start, encoding: 'utf8' });
-        const lines = createInterface({ input, crlfDelay: Infinity });
-        try {
-            for await (const line of lines) {
-                const bytes = Buffer.byteLength(line) + 1;
-                if (bytes > limits.maxRecordBytes || total + bytes > limits.maxTotalBytes
-                    || count + 1 > limits.maxRecords) {
-                    throw new SessionContractError('RESOURCE_LIMIT_EXCEEDED', 'bounded session reader limit exceeded');
-                }
-                if (line.trim() === '') {
-                    offset += bytes;
-                    total += bytes;
-                    continue;
-                }
-                let value;
-                try {
-                    value = JSON.parse(line);
-                }
-                catch {
-                    if (options.consistency === 'provisional') {
-                        incompleteTail = true;
-                        break;
-                    }
-                    throw new SessionContractError('SCHEMA_DRIFT', 'malformed JSONL record in consistent source');
-                }
-                if (jsonDepth(value) > limits.maxNestingDepth) {
-                    throw new SessionContractError('RESOURCE_LIMIT_EXCEEDED', 'JSON nesting depth exceeds reader limit');
-                }
-                const record = { value, sequence: count, byteOffset: offset, byteLength: bytes };
-                count += 1;
+        for await (const { line, byteLength: bytes } of streamJsonLines(allowed.canonicalPath, { start, limits })) {
+            assertSessionResourceLimit('maxRecords', limits.maxRecords, count + 1);
+            if (line.trim() === '') {
                 offset += bytes;
                 total += bytes;
-                yield record;
+                continue;
             }
-        }
-        finally {
-            lines.close();
-            input.destroy();
+            let value;
+            try {
+                value = JSON.parse(line);
+            }
+            catch {
+                if (options.consistency === 'provisional') {
+                    incompleteTail = true;
+                    break;
+                }
+                throw new SessionContractError('SCHEMA_DRIFT', 'malformed JSONL record in consistent source', schemaIssueDetail('<root>', 'invalid_json'));
+            }
+            assertSessionResourceLimit('maxNestingDepth', limits.maxNestingDepth, jsonDepth(value));
+            const record = { value, sequence: count, byteOffset: offset, byteLength: bytes };
+            count += 1;
+            offset += bytes;
+            total += bytes;
+            yield record;
         }
     };
     return {
@@ -83,30 +136,30 @@ export async function streamBoundedJsonLines(authorization, options) {
         [Symbol.asyncIterator]: iterate,
     };
 }
+// Single-document JSON and text exports cannot be incrementally normalized.
+export const MAX_BUFFERED_SOURCE_BYTES = 64 * 1024 * 1024;
 export async function readBoundedJson(authorization, limitsInput) {
     const allowed = await authorizeSourceFile(authorization);
     const limits = { ...DEFAULT_READER_LIMITS, ...limitsInput };
-    if (allowed.size > limits.maxTotalBytes || allowed.size > limits.maxRecordBytes) {
-        throw new SessionContractError('RESOURCE_LIMIT_EXCEEDED', 'bounded JSON source limit exceeded');
-    }
+    assertImportByteLimit('maxTotalBytes', limits, allowed.size);
+    assertImportByteLimit('maxRecordBytes', limits, allowed.size);
+    assertSessionResourceLimit('maxBufferedSourceBytes', MAX_BUFFERED_SOURCE_BYTES, allowed.size);
     let value;
     try {
         value = JSON.parse(await readFile(allowed.canonicalPath, 'utf8'));
     }
     catch {
-        throw new SessionContractError('SCHEMA_DRIFT', 'malformed JSON source');
-    }
-    if (jsonDepth(value) > limits.maxNestingDepth) {
-        throw new SessionContractError('RESOURCE_LIMIT_EXCEEDED', 'JSON nesting depth exceeds reader limit');
+        throw new SessionContractError('SCHEMA_DRIFT', 'malformed JSON source', schemaIssueDetail('<root>', 'invalid_json'));
     }
+    assertSessionResourceLimit('maxNestingDepth', limits.maxNestingDepth, jsonDepth(value));
     return { value, bytesRead: allowed.size };
 }
 export async function readBoundedText(authorization, limitsInput) {
     const allowed = await authorizeSourceFile(authorization);
     const limits = { ...DEFAULT_READER_LIMITS, ...limitsInput };
-    if (allowed.size > limits.maxTotalBytes || allowed.size > limits.maxRecordBytes) {
-        throw new SessionContractError('RESOURCE_LIMIT_EXCEEDED', 'bounded text source limit exceeded');
-    }
+    assertImportByteLimit('maxTotalBytes', limits, allowed.size);
+    assertImportByteLimit('maxRecordBytes', limits, allowed.size);
+    assertSessionResourceLimit('maxBufferedSourceBytes', MAX_BUFFERED_SOURCE_BYTES, allowed.size);
     return {
         value: await readFile(allowed.canonicalPath, 'utf8'),
         bytesRead: allowed.size,
@@ -136,7 +189,7 @@ export async function fingerprintSourcePrefix(authorization, length, skipPrefixB
     const allowed = await authorizeSourceFile(authorization);
     if (!Number.isSafeInteger(length) || length < 0 || length > allowed.size
         || !Number.isSafeInteger(skipPrefixBytes) || skipPrefixBytes < 0 || skipPrefixBytes > allowed.size) {
-        throw new SessionContractError('SCHEMA_DRIFT', 'checkpoint source position is beyond the current source size');
+        throw new SessionContractError('SCHEMA_DRIFT', 'checkpoint source position is beyond the current source size', schemaIssueDetail('cursor'));
     }
     const handle = await open(allowed.canonicalPath, 'r');
     const hash = createHash('sha256');
@@ -165,7 +218,7 @@ function parseCursor(cursor) {
     if (cursor === undefined || cursor === '')
         return 0;
     if (!/^\d+$/.test(cursor))
-        throw new SessionContractError('SCHEMA_DRIFT', 'invalid reader cursor');
+        throw new SessionContractError('SCHEMA_DRIFT', 'invalid reader cursor', schemaIssueDetail('cursor'));
     return Number(cursor);
 }
 function jsonDepth(value, depth = 0) {

package/dist/src/sessions/refresh.js

@@ -0,0 +1,106 @@
+import { existsSync, realpathSync } from 'node:fs';
+import path from 'node:path';
+import { requireFeaturePackage } from '../features/runtime.js';
+import { readAiwgConfig } from '../config/aiwg-config.js';
+import { createSessionAdapterRegistry } from './adapter-registry.js';
+import { resolveImportExclusions } from './import-exclusions.js';
+import { resolveImportLimits } from './import-limits.js';
+import { acquireImportLease } from './import-lease.js';
+import { SessionRepository } from './repository.js';
+import { importDiscoveryManifest } from './batch-import.js';
+import { defaultDiscoveryManifestPath, discoverWorkspaceHistories, readDiscoveryManifest, writeDiscoveryManifest, } from './workspace-discovery.js';
+export function sessionRefreshAvailable() {
+    try {
+        const Database = requireFeaturePackage('better-sqlite3');
+        const probe = new Database(':memory:');
+        probe.close();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Refresh an existing catalog without inferring or granting provider access. */
+export async function refreshSessionCatalog(workspace, full, waitMs, signal, workspaceAliases = [], flags = {}) {
+    const canonicalWorkspace = realpathSync(workspace);
+    const config = await readAiwgConfig(canonicalWorkspace);
+    const limits = resolveImportLimits(config?.sessions?.import, flags);
+    const exclude = resolveImportExclusions(config?.sessions?.import, flags.exclude);
+    const database = path.join(workspace, '.aiwg/sessions/catalog.sqlite');
+    const manifests = [];
+    if (!full) {
+        const paths = new Set([
+            defaultDiscoveryManifestPath(canonicalWorkspace),
+            ...(config?.sessions?.discovery_manifests ?? []).map(file => path.resolve(canonicalWorkspace, file)),
+        ]);
+        let found = false;
+        for (const manifestPath of paths) {
+            if (!existsSync(manifestPath))
+                continue;
+            const previous = await readDiscoveryManifest(manifestPath);
+            if (previous.workspaceId !== canonicalWorkspace)
+                throw new Error('discovery manifest workspace mismatch');
+            found = true;
+            const authorizedRoots = previous.authorizedRoots ?? {};
+            if (!previous.authorizedRoots) {
+                for (const source of previous.sources) {
+                    const roots = authorizedRoots[source.provider] ??= [];
+                    if (!roots.includes(source.authorizedRoot))
+                        roots.push(source.authorizedRoot);
+                }
+            }
+            const providers = Object.keys(authorizedRoots)
+                .filter(provider => authorizedRoots[provider]?.length);
+            if (!providers.length)
+                continue;
+            const manifest = await discoverWorkspaceHistories({
+                workspace: canonicalWorkspace,
+                workspaceAliases: [workspace, ...workspaceAliases],
+                expectedProviders: providers,
+                authorizedRoots,
+                registry: createSessionAdapterRegistry(limits),
+                exclude,
+                knownSources: previous.sources,
+            });
+            manifests.push({ path: manifestPath, manifest });
+        }
+        if (!found)
+            return 'no authorized discovery manifest';
+        if (!manifests.length)
+            return 'no authorized provider roots';
+    }
+    const lease = await acquireImportLease(database, `index-refresh-${process.pid}`, { waitMs });
+    try {
+        const repository = new SessionRepository(database);
+        try {
+            if (full)
+                repository.reindex();
+            else {
+                const receipts = [];
+                for (const entry of manifests) {
+                    const prior = repository.getBatchImportRunForManifest(entry.manifest.manifestId, canonicalWorkspace);
+                    if (prior?.status === 'complete')
+                        continue;
+                    // Persist the exact discovered batch before its confirmed import.
+                    await writeDiscoveryManifest(entry.path, entry.manifest);
+                    const receipt = await importDiscoveryManifest({ manifest: entry.manifest, repository, signal, limits, exclude });
+                    if (receipt.totals.pending) {
+                        throw new Error(`session import incomplete: ${receipt.totals.rejected} rejected, ${receipt.totals.pending} pending`);
+                    }
+                    receipts.push(receipt);
+                }
+                if (!receipts.length)
+                    return 'unchanged';
+                return receipts.length === 1 ? receipts[0] : receipts;
+            }
+        }
+        finally {
+            repository.close();
+        }
+    }
+    finally {
+        await lease.release();
+    }
+    return undefined;
+}
+//# sourceMappingURL=refresh.js.map