@aiwg/cli 2026.10.10 → 2026.10.11

package/dist/src/sessions/host-catalog.js

@@ -0,0 +1,157 @@
+// Host catalog for account-scoped session providers (#3232).
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { createSessionAdapterRegistry } from './adapter-registry.js';
+/** Workspace id used for account-scoped sessions stored in the host catalog. */
+export const HOST_CATALOG_WORKSPACE_ID = 'host';
+/** DDL for the workspace-catalog reference table (created by SessionRepository). */
+export const HOST_REFERENCE_TABLE_DDL = `
+  CREATE TABLE IF NOT EXISTS session_host_refs (
+    provider TEXT NOT NULL, native_session_id TEXT NOT NULL,
+    session_id TEXT NOT NULL, source_id TEXT NOT NULL,
+    host_catalog TEXT NOT NULL,
+    PRIMARY KEY(provider, native_session_id)
+  );
+  CREATE INDEX IF NOT EXISTS idx_session_host_refs_session
+  ON session_host_refs(session_id);
+`;
+const HOST_CATALOG_ENV = 'AIWG_SESSIONS_HOST_CATALOG';
+/** User-scope (host) catalog path; the env override exists so tests never touch HOME. */
+export function defaultHostCatalogPath(homeDir = homedir()) {
+    const override = process.env[HOST_CATALOG_ENV]?.trim();
+    if (override)
+        return override;
+    return join(homeDir, '.aiwg', 'sessions', 'host-catalog.sqlite');
+}
+/** Account-scoped adapters (Grok Bot today) route to the host catalog. */
+export function isAccountScopedAdapter(adapter) {
+    return adapter.scopeClass === 'account';
+}
+/**
+ * Whether a discovery manifest holds any account-scoped source. Callers use
+ * this to avoid creating or opening the host catalog when every source is
+ * workspace-scoped. Unknown providers resolve to workspace scope.
+ */
+export function manifestNeedsHostCatalog(sources) {
+    if (sources.length === 0)
+        return false;
+    const registry = createSessionAdapterRegistry();
+    return sources.some((source) => {
+        try {
+            return isAccountScopedAdapter(registry.get(source.provider));
+        }
+        catch {
+            return false;
+        }
+    });
+}
+/** Aggregate identity: one account-scoped conversation counts once per provider. */
+export function hostReferenceKey(identity) {
+    return `${identity.provider}\0${identity.nativeSessionId}`;
+}
+/** Drop cross-catalog duplicates, keeping the first row for each provider+session. */
+export function dedupeSessionIdentities(rows) {
+    const seen = new Set();
+    return rows.filter((row) => {
+        const key = hostReferenceKey(row);
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+/** Count sessions across catalogs without doubling account-scoped replicas. */
+export function countUniqueSessionsAcross(lists) {
+    return dedupeSessionIdentities(lists.flat()).length;
+}
+/** Host sessions backing a workspace catalog's references (matched by provider+session). */
+export function resolveHostReferences(references, hostSessions) {
+    const byKey = new Map(hostSessions.map((session) => [
+        hostReferenceKey({ provider: session.provider, nativeSessionId: session.nativeSessionId }),
+        session,
+    ]));
+    const resolved = [];
+    for (const reference of references) {
+        const session = byKey.get(hostReferenceKey(reference));
+        if (session)
+            resolved.push(session);
+    }
+    return resolved;
+}
+/**
+ * Workspace listing view: local rows plus referenced host sessions, with
+ * legacy duplicated copies (same provider+session already stored locally)
+ * counted once. Local rows win ties so workspace provenance is preserved.
+ */
+export function mergeSessionLists(local, referenced) {
+    return dedupeSessionIdentities([...local, ...referenced]);
+}
+/** Sessions in a repository reduced to aggregate identities (legacy rows included). */
+export function collectSessionIdentities(repository, workspaceId) {
+    return repository.listSessionIdentities(workspaceId);
+}
+/**
+ * Count sessions across workspace and host catalogs without doubling
+ * account-scoped replicas: two workspaces referencing the same host
+ * session plus a legacy workspace holding a copied row count once.
+ */
+export function countUniqueSessionsAcrossCatalogs(views) {
+    return dedupeSessionIdentities(views.flatMap((view) => [...view.identities, ...view.references])).length;
+}
+/** Index visible identities by internal session id for merge lookups. */
+export function indexIdentitiesBySessionId(identities) {
+    return new Map(identities.map((identity) => [identity.sessionId, identity]));
+}
+/**
+ * Merge local rows (search hits, analytics facts) with host-catalog rows,
+ * counting each session once by provider + native session id. Every local
+ * row is kept; a host row is dropped when the same session already has
+ * local rows (a legacy copied row wins over its host reference, mirroring
+ * mergeSessionLists, #3232) and host rows repeating across catalogs are
+ * deduped by hostRowKey. An unreachable host catalog contributes no rows,
+ * so local results are never lost.
+ */
+export function mergeCrossCatalogRows(localRows, hostRows, localIdentityBySessionId, hostIdentityBySessionId, hostRowKey) {
+    const localKeys = new Set();
+    for (const row of localRows) {
+        const identity = localIdentityBySessionId.get(row.sessionId);
+        if (identity) {
+            localKeys.add(hostReferenceKey({ provider: row.provider, nativeSessionId: identity.nativeSessionId }));
+        }
+    }
+    const seenHost = new Set();
+    const keptHost = hostRows.filter((row) => {
+        const identity = hostIdentityBySessionId.get(row.sessionId);
+        if (identity
+            && localKeys.has(hostReferenceKey({ provider: row.provider, nativeSessionId: identity.nativeSessionId }))) {
+            return false;
+        }
+        const key = hostRowKey(row);
+        if (seenHost.has(key))
+            return false;
+        seenHost.add(key);
+        return true;
+    });
+    return [...localRows, ...keptHost];
+}
+/**
+ * Publish staged account-scoped imports in the host catalog and record
+ * workspace references instead of copies (#3232). Non-destructive: legacy
+ * workspace rows are left untouched and stay readable.
+ */
+export function publishHostRoutedBatch(input) {
+    const accepted = new Set(input.sourceIds);
+    input.hostRepository.commitStagedBatch(input.batchRunId, [...accepted]);
+    for (const identity of input.hostRepository.listSessionIdentities(HOST_CATALOG_WORKSPACE_ID)) {
+        if (!accepted.has(identity.sourceId))
+            continue;
+        input.workspaceRepository.recordHostSessionReference({
+            provider: identity.provider,
+            nativeSessionId: identity.nativeSessionId,
+            sessionId: identity.sessionId,
+            sourceId: identity.sourceId,
+            hostCatalog: input.hostCatalogPath,
+        });
+    }
+}
+//# sourceMappingURL=host-catalog.js.map

package/dist/src/sessions/image-blobs.js

@@ -0,0 +1,141 @@
+// Content-addressed tool-output image references for the session catalog (#3265).
+import { createHash } from 'node:crypto';
+const IMAGE_DIGEST_PREFIX = 'sha256:';
+export function sha256Bytes(value) {
+    return `${IMAGE_DIGEST_PREFIX}${createHash('sha256').update(value).digest('hex')}`;
+}
+export function isImageDigest(value) {
+    return typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value);
+}
+/** Human-readable byte size for image placeholders (`3.1 MB`, `68 B`). */
+export function formatImageBytes(byteLength) {
+    if (!Number.isFinite(byteLength) || byteLength < 0)
+        return '0 B';
+    if (byteLength < 1024)
+        return `${Math.trunc(byteLength)} B`;
+    const kb = byteLength / 1024;
+    if (kb < 1024)
+        return `${kb.toFixed(1)} KB`;
+    return `${(kb / 1024).toFixed(1)} MB`;
+}
+/** Placeholder kept in `searchableText` (and FTS) instead of the image bytes. */
+export function imagePlaceholder(ref) {
+    return `[image: ${ref.digest}, ${ref.mediaType}, ${formatImageBytes(ref.byteLength)}]`;
+}
+// The payload class excludes whitespace so a match stops at the first word
+// boundary instead of swallowing following prose; shape (and a minimum
+// length well below any real image) is validated again on decode.
+const DATA_URL_PATTERN = /data:(image\/[a-zA-Z0-9+.-]+)(;[a-zA-Z0-9!#$&^_.+-]+)*;base64,([A-Za-z0-9+/=]+)/g;
+const MIN_BASE64_PAYLOAD_CHARS = 32;
+function decodeBase64Payload(payload) {
+    const compact = payload.replace(/\s+/g, '');
+    // `Buffer.from(x, 'base64')` never throws -- it skips invalid characters --
+    // so validate the shape first: base64 alphabet only, padding at the end,
+    // and a length that can carry at least one byte.
+    if (compact.length < 4 || compact.length % 4 === 1)
+        return null;
+    if (!/^(?:[A-Za-z0-9+/]+={0,2})$/.test(compact))
+        return null;
+    const bytes = Buffer.from(compact, 'base64');
+    if (bytes.length === 0)
+        return null;
+    return new Uint8Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength));
+}
+/**
+ * Replace inline base64 image data URLs in free text with content-hash
+ * placeholders. Only `image/*` media types are extracted; other data URLs
+ * (and remote `http(s)` image URLs, which are references already) pass
+ * through untouched.
+ */
+export function extractInlineImagesFromText(text, source = {}) {
+    const images = [];
+    if (!text.includes('data:image/'))
+        return { text, images };
+    DATA_URL_PATTERN.lastIndex = 0;
+    const result = text.replace(DATA_URL_PATTERN, (match, mediaType, _params, payload) => {
+        if (payload.length < MIN_BASE64_PAYLOAD_CHARS)
+            return match;
+        const bytes = decodeBase64Payload(payload);
+        if (!bytes)
+            return match;
+        const ref = {
+            digest: sha256Bytes(bytes),
+            mediaType: String(mediaType).toLowerCase(),
+            byteLength: bytes.length,
+            ...(source.locatorClass !== undefined ? { locatorClass: source.locatorClass } : {}),
+            ...(source.offset !== undefined ? { offset: source.offset } : {}),
+            bytes,
+        };
+        images.push(ref);
+        return imagePlaceholder(ref);
+    });
+    return { text: result, images };
+}
+function isPlainObject(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+/**
+ * Extract inline images from a whole import batch. Returns replacement
+ * events (bytes swapped for references) plus the distinct blobs to persist.
+ * Idempotent: placeholder text from an earlier pass matches no pattern, so
+ * importer-extracted batches and direct writers converge on the same store
+ * state. Blobs travel beside the events because event JSON must only hold
+ * the content-hash reference, never the bytes.
+ */
+export function extractBatchEventImages(batch) {
+    const blobs = new Map();
+    for (const carried of batch.imageBlobs ?? []) {
+        if (isImageDigest(carried.digest) && !blobs.has(carried.digest))
+            blobs.set(carried.digest, carried);
+    }
+    const events = batch.events.map((event) => {
+        const extracted = extractEventImages({
+            searchableText: event.searchableText,
+            extensions: event.extensions,
+            rawReference: event.rawReference,
+        });
+        for (const image of extracted.images) {
+            if (!blobs.has(image.digest))
+                blobs.set(image.digest, image);
+        }
+        if (extracted.images.length === 0)
+            return event;
+        const refs = [...(event.imageRefs ?? [])];
+        for (const ref of extracted.imageRefs) {
+            if (!refs.some((prior) => prior.digest === ref.digest))
+                refs.push(ref);
+        }
+        return {
+            ...event,
+            searchableText: extracted.searchableText,
+            extensions: extracted.extensions,
+            imageRefs: refs,
+        };
+    });
+    return { events, imageBlobs: [...blobs.values()] };
+}
+/** Extract inline images from one event's text; extensions keep their existing redaction. */
+export function extractEventImages(input) {
+    const source = {
+        ...(input.rawReference?.locatorClass !== undefined ? { locatorClass: input.rawReference.locatorClass } : {}),
+        ...(input.rawReference?.offset !== undefined ? { offset: input.rawReference.offset } : {}),
+    };
+    // Only event text is lifted. Provider image blocks in `extensions` are
+    // left to native-extension sanitization, which redacts them to markers,
+    // so this change never retains image bytes the catalog used to drop.
+    const text = extractInlineImagesFromText(input.searchableText ?? '', source);
+    const images = text.images;
+    return {
+        searchableText: text.text,
+        extensions: isPlainObject(input.extensions) ? input.extensions : {},
+        imageRefs: images.map((image) => ({
+            digest: image.digest,
+            mediaType: image.mediaType,
+            byteLength: image.byteLength,
+            ...(image.locatorClass !== undefined ? { locatorClass: image.locatorClass } : {}),
+            ...(image.offset !== undefined ? { offset: image.offset } : {}),
+        })),
+        images,
+    };
+}
+//# sourceMappingURL=image-blobs.js.map

package/dist/src/sessions/importer.js

@@ -1,6 +1,7 @@
 import { SESSION_CONTRACT_VERSION, SessionEventSchema, SessionSchema, assertSupportedSchemaMajor, sha256, stableEventId, stableSessionId, SessionContractError, schemaIssueDetail, assertSessionResourceLimit, } from './contracts.js';
 import { assertImportByteLimit, DEFAULT_IMPORT_BYTE_LIMITS, resolveImportLimits } from './import-limits.js';
 import { redactSessionValue, sanitizeSessionEvent, sanitizeNativeExtensions } from './policy.js';
+import { extractEventImages } from './image-blobs.js';
 import { fingerprintSourcePrefix } from './readers.js';
 import { DEFAULT_INACTIVITY_THRESHOLD_MS } from './lifecycle.js';
 import { classifySessionEventOrigin, deriveSessionIntent, } from './origin.js';
@@ -123,6 +124,7 @@ export class IncrementalSessionImporter {
                 source: sanitizedSource, run,
                 sessions: normalized.sessions, events: normalized.events,
                 batchRunId: request.batchRunId,
+                imageBlobs: normalized.imageBlobs,
             }, { ...checkpoint }, request.publish ?? request.source.consistency !== 'complete');
             const memory = process.memoryUsage();
             peakHeap = Math.max(peakHeap, memory.heapUsed);
@@ -232,17 +234,30 @@ function recordIdentity(record) {
 }
 function normalizeBatch(request, records, digests, importRunId, sourceObservedAt) {
     const sessions = new Map();
+    // Extract inline images before sanitization: provider image blocks in
+    // extensions are redacted to markers below, so bytes must be lifted out
+    // first while the raw record is still intact (#3265).
+    const imageBlobs = new Map();
     const events = records.map((record, index) => {
         const sessionId = stableSessionId(request.source.provider, request.source.sourceId, record.nativeSessionId);
         const native = sanitizeNativeExtensions(record.extensions ?? {});
         const lineage = request.source.provider === 'codex' && native.value.lineage
             ? { lineage: native.value.lineage } : {};
+        const images = extractEventImages({
+            searchableText: record.text,
+            extensions: record.extensions,
+            rawReference: record.rawReference,
+        });
+        for (const image of images.images) {
+            if (!imageBlobs.has(image.digest))
+                imageBlobs.set(image.digest, image);
+        }
         const digest = digests[index];
         const origin = classifySessionEventOrigin(request.source.provider, record);
         const lifecycle = sessionLifecycle(request, record, sourceObservedAt);
         const lifecycleEvidence = sessionLifecycleEvidence(request, record, sourceObservedAt);
         const extensions = {
-            [`native.${request.source.provider}`]: record.extensions ?? {},
+            [`native.${request.source.provider}`]: images.extensions,
         };
         const event = sanitizeSessionEvent(SessionEventSchema.parse({
             contractVersion: SESSION_CONTRACT_VERSION,
@@ -262,7 +277,8 @@ function normalizeBatch(request, records, digests, importRunId, sourceObservedAt
             model: record.model ?? null,
             entities: record.entities ?? [],
             extractionState: record.extractionState ?? null,
-            searchableText: record.text, digest, rawReference: record.rawReference,
+            searchableText: images.searchableText, imageRefs: images.imageRefs,
+            digest, rawReference: record.rawReference,
             adapterVersion: request.adapter.adapterVersion,
             consistency: request.source.consistency,
             sensitivity: { classification: 'none', classes: [] },
@@ -309,7 +325,11 @@ function normalizeBatch(request, records, digests, importRunId, sourceObservedAt
     for (const session of sessions.values()) {
         session.intent = deriveSessionIntent(events.filter((event) => event.sessionId === session.sessionId));
     }
-    return { sessions: redactSessionValue([...sessions.values()]).value, events };
+    return {
+        sessions: redactSessionValue([...sessions.values()]).value,
+        events,
+        imageBlobs: [...imageBlobs.values()],
+    };
 }
 const KNOWN_EVENT_KINDS = new Set(['message', 'tool-call', 'tool-result', 'artifact', 'attachment', 'summary']);
 function isKnownEventKind(kind) {

package/dist/src/sessions/index.js

@@ -1,4 +1,5 @@
 export * from './contracts.js';
+export * from './image-blobs.js';
 export * from './fixtures.js';
 export * from './policy.js';
 export * from './discovery.js';
@@ -19,6 +20,7 @@ export * from './importer.js';
 export * from './import-lease.js';
 export * from './batch-contracts.js';
 export * from './batch-import.js';
+export * from './host-catalog.js';
 export * from './workspace-discovery.js';
 export * from './timeline.js';
 export * from './origin.js';

package/dist/src/sessions/refresh.js

@@ -1,4 +1,4 @@
-import { existsSync, realpathSync } from 'node:fs';
+import { existsSync, mkdirSync, realpathSync } from 'node:fs';
 import path from 'node:path';
 import { requireFeaturePackage } from '../features/runtime.js';
 import { readAiwgConfig } from '../config/aiwg-config.js';
@@ -6,6 +6,7 @@ import { createSessionAdapterRegistry } from './adapter-registry.js';
 import { resolveImportExclusions } from './import-exclusions.js';
 import { resolveImportLimits } from './import-limits.js';
 import { acquireImportLease } from './import-lease.js';
+import { defaultHostCatalogPath, manifestNeedsHostCatalog } from './host-catalog.js';
 import { SessionRepository } from './repository.js';
 import { importDiscoveryManifest } from './batch-import.js';
 import { defaultDiscoveryManifestPath, discoverWorkspaceHistories, readDiscoveryManifest, writeDiscoveryManifest, } from './workspace-discovery.js';
@@ -69,8 +70,26 @@ export async function refreshSessionCatalog(workspace, full, waitMs, signal, wor
             return 'no authorized provider roots';
     }
     const lease = await acquireImportLease(database, `index-refresh-${process.pid}`, { waitMs });
+    let hostLease;
     try {
         const repository = new SessionRepository(database);
+        // Account-scoped providers import once into the host catalog; the
+        // hourly host refresh (#310) routes them there instead of copying
+        // them into every workspace catalog (#3232). The host catalog is only
+        // opened when a manifest actually holds an account-scoped source.
+        const hostCatalogPath = defaultHostCatalogPath();
+        const needsHost = manifests.some((entry) => manifestNeedsHostCatalog(entry.manifest.sources));
+        let hostRepository;
+        if (needsHost) {
+            mkdirSync(path.dirname(hostCatalogPath), { recursive: true, mode: 0o700 });
+            // The workspace lease above does not cover the shared host catalog,
+            // so overlapping refreshes from two workspaces would stage the same
+            // replicas concurrently (#3232). Serialize host writers with the
+            // same lock pattern; the shared checkpoint then turns the second
+            // writer into duplicate references instead of a double import.
+            hostLease = await acquireImportLease(hostCatalogPath, `host-refresh-${process.pid}`, { waitMs });
+            hostRepository = new SessionRepository(hostCatalogPath);
+        }
         try {
             if (full)
                 repository.reindex();
@@ -82,7 +101,9 @@ export async function refreshSessionCatalog(workspace, full, waitMs, signal, wor
                         continue;
                     // Persist the exact discovered batch before its confirmed import.
                     await writeDiscoveryManifest(entry.path, entry.manifest);
-                    const receipt = await importDiscoveryManifest({ manifest: entry.manifest, repository, signal, limits, exclude });
+                    const receipt = await importDiscoveryManifest({
+                        manifest: entry.manifest, repository, hostRepository, hostCatalogPath, signal, limits, exclude,
+                    });
                     if (receipt.totals.pending) {
                         throw new Error(`session import incomplete: ${receipt.totals.rejected} rejected, ${receipt.totals.pending} pending`);
                     }
@@ -94,10 +115,12 @@ export async function refreshSessionCatalog(workspace, full, waitMs, signal, wor
             }
         }
         finally {
+            hostRepository?.close();
             repository.close();
         }
     }
     finally {
+        await hostLease?.release();
         await lease.release();
     }
     return undefined;