@jungjaehoon/mama-core 1.4.2 → 1.6.0

package/dist/connectors/raw-query.js

@@ -0,0 +1,305 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.searchRaw = searchRaw;
+exports.searchAllRaw = searchAllRaw;
+exports.getRawById = getRawById;
+exports.getRawWindow = getRawWindow;
+const types_js_1 = require("../memory/types.js");
+const event_index_js_1 = require("./event-index.js");
+const DEFAULT_RAW_LIMIT = 25;
+const MAX_RAW_LIMIT = 100;
+const DEFAULT_WINDOW_SIZE = 5;
+const MAX_WINDOW_SIZE = 50;
+const VALID_SCOPE_KINDS = new Set(types_js_1.MEMORY_SCOPE_KINDS);
+function placeholders(values) {
+    if (values.length === 0) {
+        throw new Error('Cannot build SQL placeholders for an empty list.');
+    }
+    return values.map(() => '?').join(', ');
+}
+function normalizeLimit(value) {
+    if (value === undefined) {
+        return DEFAULT_RAW_LIMIT;
+    }
+    return Math.min(MAX_RAW_LIMIT, Math.max(0, Math.floor(value)));
+}
+function normalizeWindowSize(value) {
+    if (value === undefined) {
+        return DEFAULT_WINDOW_SIZE;
+    }
+    return Math.min(MAX_WINDOW_SIZE, Math.max(0, Math.floor(value)));
+}
+function escapeFtsQuery(query) {
+    return `"${query.replace(/"/g, '""')}"`;
+}
+function encodeCursor(row) {
+    return Buffer.from(JSON.stringify({
+        rank: row.rank,
+        timestampMs: row.source_timestamp_ms,
+        rawId: row.event_index_id,
+    }), 'utf8').toString('base64url');
+}
+function decodeCursor(cursor) {
+    if (!cursor) {
+        return null;
+    }
+    try {
+        const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
+        if (typeof parsed.rank !== 'number' ||
+            !Number.isFinite(parsed.rank) ||
+            typeof parsed.timestampMs !== 'number' ||
+            !Number.isFinite(parsed.timestampMs) ||
+            typeof parsed.rawId !== 'string' ||
+            parsed.rawId.length === 0) {
+            throw new Error('invalid shape');
+        }
+        return {
+            rank: parsed.rank,
+            timestampMs: parsed.timestampMs,
+            rawId: parsed.rawId,
+        };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Invalid raw search cursor: ${message}`);
+    }
+}
+function normalizeConnectors(connectors) {
+    return [...new Set((connectors ?? []).map((connector) => connector.trim()).filter(Boolean))];
+}
+function normalizeScopes(scopes) {
+    return (scopes ?? [])
+        .map((scope) => {
+        const kind = scope.kind.trim();
+        if (!VALID_SCOPE_KINDS.has(kind)) {
+            throw new Error(`Invalid raw search scope kind: ${kind}`);
+        }
+        return { kind: kind, id: scope.id.trim() };
+    })
+        .filter((scope) => scope.id.length > 0);
+}
+function appendFilters(clauses, params, input, alias = 'e') {
+    const connectors = normalizeConnectors(input.connectors);
+    if (connectors.length > 0) {
+        clauses.push(`${alias}.source_connector IN (${placeholders(connectors)})`);
+        params.push(...connectors);
+    }
+    const scopes = normalizeScopes(input.scopes);
+    if (scopes.length > 0) {
+        clauses.push(`(${scopes.map(() => `(${alias}.memory_scope_kind = ? AND ${alias}.memory_scope_id = ?)`).join(' OR ')})`);
+        for (const scope of scopes) {
+            params.push(scope.kind, scope.id);
+        }
+    }
+    if (input.fromMs !== undefined) {
+        clauses.push(`${alias}.source_timestamp_ms >= ?`);
+        params.push(input.fromMs);
+    }
+    if (input.toMs !== undefined) {
+        clauses.push(`${alias}.source_timestamp_ms <= ?`);
+        params.push(input.toMs);
+    }
+}
+function appendCursorFilter(clauses, params, cursor) {
+    if (!cursor) {
+        return;
+    }
+    clauses.push(`(rank > ? OR (rank = ? AND source_timestamp_ms < ?) OR ` +
+        `(rank = ? AND source_timestamp_ms = ? AND event_index_id > ?))`);
+    params.push(cursor.rank, cursor.rank, cursor.timestampMs, cursor.rank, cursor.timestampMs, cursor.rawId);
+}
+function parseMetadata(metadataJson) {
+    if (!metadataJson) {
+        return {};
+    }
+    try {
+        const parsed = JSON.parse(metadataJson);
+        return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+            ? parsed
+            : {};
+    }
+    catch {
+        return {};
+    }
+}
+function contentPreview(content) {
+    const compact = content.replace(/\s+/g, ' ').trim();
+    return compact.length > 240 ? `${compact.slice(0, 237)}...` : compact;
+}
+function scoreFromRank(rank) {
+    return Number.isFinite(rank) ? 1 / (1 + Math.exp(rank)) : 0;
+}
+function timestampToIso(timestampMs) {
+    if (timestampMs === null || !Number.isFinite(timestampMs)) {
+        return null;
+    }
+    return new Date(timestampMs).toISOString();
+}
+function toRawHit(row) {
+    const record = (0, event_index_js_1.mapConnectorEventIndexRecord)(row);
+    return {
+        raw_id: record.event_index_id,
+        connector: record.source_connector,
+        source_id: record.source_id,
+        channel_id: record.channel,
+        author_label: record.author,
+        created_at: timestampToIso(record.event_datetime ?? record.source_timestamp_ms),
+        content_preview: contentPreview(record.content),
+        score: scoreFromRank(Number(row.rank)),
+        source_ref: record.source_locator ?? record.artifact_locator,
+        metadata: parseMetadata(record.metadata_json),
+    };
+}
+function runSearch(adapter, input) {
+    const query = input.query.trim();
+    if (query.length === 0) {
+        return { hits: [], next_cursor: null };
+    }
+    const limit = normalizeLimit(input.limit);
+    if (limit === 0) {
+        return { hits: [], next_cursor: null };
+    }
+    const params = [escapeFtsQuery(query)];
+    const clauses = [];
+    appendFilters(clauses, params, input);
+    const cursor = decodeCursor(input.cursor);
+    const outerClauses = [];
+    const outerParams = [];
+    appendCursorFilter(outerClauses, outerParams, cursor);
+    const whereSql = clauses.length > 0 ? `AND ${clauses.join(' AND ')}` : '';
+    const cursorSql = outerClauses.length > 0 ? `WHERE ${outerClauses.join(' AND ')}` : '';
+    const rows = adapter
+        .prepare(`
+        WITH ranked AS (
+          SELECT e.*, bm25(connector_event_index_fts) AS rank
+          FROM connector_event_index_fts
+          JOIN connector_event_index e
+            ON e.event_index_id = connector_event_index_fts.event_index_id
+          WHERE connector_event_index_fts MATCH ?
+            ${whereSql}
+        )
+        SELECT *
+        FROM ranked
+        ${cursorSql}
+        ORDER BY rank ASC, source_timestamp_ms DESC, event_index_id ASC
+        LIMIT ?
+      `)
+        .all(...params, ...outerParams, limit + 1);
+    const pageRows = rows.slice(0, limit);
+    const nextRow = rows.length > limit ? pageRows[pageRows.length - 1] : undefined;
+    return {
+        hits: pageRows.map(toRawHit),
+        next_cursor: nextRow
+            ? encodeCursor({
+                rank: Number(nextRow.rank),
+                source_timestamp_ms: Number(nextRow.source_timestamp_ms),
+                event_index_id: String(nextRow.event_index_id),
+            })
+            : null,
+    };
+}
+function searchRaw(adapter, input) {
+    const connectors = normalizeConnectors(input.connectors);
+    if (connectors.length !== 1) {
+        throw new Error('searchRaw requires exactly one connector filter.');
+    }
+    return runSearch(adapter, { ...input, connectors });
+}
+function searchAllRaw(adapter, input) {
+    return runSearch(adapter, input);
+}
+function getRawById(adapter, rawId, visibility) {
+    const params = [rawId];
+    const clauses = ['e.event_index_id = ?'];
+    appendFilters(clauses, params, { query: '*', ...visibility });
+    const row = adapter
+        .prepare(`
+        SELECT e.*, 0 AS rank
+        FROM connector_event_index e
+        WHERE ${clauses.join(' AND ')}
+        LIMIT 1
+      `)
+        .get(...params);
+    return row ? toRawHit(row) : null;
+}
+function getRawWindow(adapter, rawId, input) {
+    const targetParams = [rawId];
+    const targetClauses = ['e.event_index_id = ?'];
+    appendFilters(targetClauses, targetParams, {
+        query: '*',
+        connectors: input.connectors,
+        scopes: input.scopes,
+    });
+    const targetRow = adapter
+        .prepare(`
+        SELECT e.*, 0 AS rank
+        FROM connector_event_index e
+        WHERE ${targetClauses.join(' AND ')}
+        LIMIT 1
+      `)
+        .get(...targetParams);
+    if (!targetRow) {
+        return null;
+    }
+    const target = (0, event_index_js_1.mapConnectorEventIndexRecord)(targetRow);
+    const before = normalizeWindowSize(input.before);
+    const after = normalizeWindowSize(input.after);
+    const beforeRows = beforeWindowRows(adapter, target, input, before);
+    const afterRows = afterWindowRows(adapter, target, input, after);
+    const targetHit = toRawHit(targetRow);
+    return {
+        target: targetHit,
+        items: [...beforeRows.reverse().map(toRawHit), targetHit, ...afterRows.map(toRawHit)],
+    };
+}
+function beforeWindowRows(adapter, target, input, limit) {
+    if (limit === 0) {
+        return [];
+    }
+    const params = [target.source_connector, target.channel, target.source_timestamp_ms];
+    const clauses = [
+        'e.source_connector = ?',
+        target.channel === null ? 'e.channel IS ?' : 'e.channel = ?',
+        'e.source_timestamp_ms < ?',
+    ];
+    appendFilters(clauses, params, {
+        query: '*',
+        connectors: input.connectors,
+        scopes: input.scopes,
+    });
+    return adapter
+        .prepare(`
+        SELECT e.*, 0 AS rank
+        FROM connector_event_index e
+        WHERE ${clauses.join(' AND ')}
+        ORDER BY e.source_timestamp_ms DESC, e.event_index_id DESC
+        LIMIT ?
+      `)
+        .all(...params, limit);
+}
+function afterWindowRows(adapter, target, input, limit) {
+    if (limit === 0) {
+        return [];
+    }
+    const params = [target.source_connector, target.channel, target.source_timestamp_ms];
+    const clauses = [
+        'e.source_connector = ?',
+        target.channel === null ? 'e.channel IS ?' : 'e.channel = ?',
+        'e.source_timestamp_ms > ?',
+    ];
+    appendFilters(clauses, params, {
+        query: '*',
+        connectors: input.connectors,
+        scopes: input.scopes,
+    });
+    return adapter
+        .prepare(`
+        SELECT e.*, 0 AS rank
+        FROM connector_event_index e
+        WHERE ${clauses.join(' AND ')}
+        ORDER BY e.source_timestamp_ms ASC, e.event_index_id ASC
+        LIMIT ?
+      `)
+        .all(...params, limit);
+}
+//# sourceMappingURL=raw-query.js.map

package/dist/connectors/types.d.ts

@@ -0,0 +1,109 @@
+import type { Buffer } from 'node:buffer';
+import type { MemoryScopeRef } from '../memory/types.js';
+export interface ConnectorEventIndexRecord {
+    event_index_id: string;
+    source_connector: string;
+    source_type: string;
+    source_id: string;
+    source_locator: string | null;
+    channel: string | null;
+    author: string | null;
+    title: string | null;
+    content: string;
+    event_datetime: number | null;
+    event_date: string | null;
+    source_timestamp_ms: number;
+    source_cursor: string | null;
+    tenant_id: string | null;
+    project_id: string | null;
+    memory_scope_kind: string | null;
+    memory_scope_id: string | null;
+    metadata_json: string | null;
+    artifact_locator: string | null;
+    artifact_title: string | null;
+    content_hash: Buffer;
+    indexed_at: string;
+    updated_at: string;
+    expires_at: string | null;
+}
+export interface UpsertConnectorEventIndexInput {
+    source_connector: string;
+    source_type: string;
+    source_id: string;
+    source_locator?: string | null;
+    channel?: string | null;
+    author?: string | null;
+    title?: string | null;
+    content: string;
+    event_datetime?: number | null;
+    event_date?: string | null;
+    source_timestamp_ms?: number | null;
+    source_cursor?: string | null;
+    tenant_id?: string | null;
+    project_id?: string | null;
+    memory_scope_kind?: string | null;
+    memory_scope_id?: string | null;
+    metadata_json?: string | null;
+    metadata?: unknown;
+    artifact_locator?: string | null;
+    artifact_title?: string | null;
+    content_hash?: Buffer | Uint8Array | null;
+    indexed_at?: string;
+    updated_at?: string;
+    expires_at?: string | null;
+}
+export interface ConnectorEventIndexCursorRecord {
+    connector_name: string;
+    last_seen_timestamp_ms: number;
+    last_seen_source_id: string;
+    last_sweep_at: string | null;
+    last_success_at: string | null;
+    last_error: string | null;
+    last_error_at: string | null;
+    indexed_count: number;
+}
+export interface UpsertConnectorEventIndexCursorInput {
+    connector_name: string;
+    last_seen_timestamp_ms?: number;
+    last_seen_source_id?: string;
+    last_sweep_at?: string | null;
+    last_success_at?: string | null;
+    last_error?: string | null;
+    last_error_at?: string | null;
+    indexed_count?: number;
+}
+export interface ConnectorEventSearchHit extends ConnectorEventIndexRecord {
+    rank: number;
+    score: number;
+}
+export interface RawSearchScopeFilter {
+    kind: MemoryScopeRef['kind'];
+    id: string;
+}
+export interface RawSearchInput {
+    query: string;
+    connectors?: string[];
+    scopes?: RawSearchScopeFilter[];
+    fromMs?: number;
+    toMs?: number;
+    cursor?: string;
+    limit?: number;
+}
+export interface RawSearchHit {
+    raw_id: string;
+    connector: string;
+    source_id: string;
+    channel_id: string | null;
+    author_label: string | null;
+    created_at: string | null;
+    content_preview: string;
+    score: number;
+    source_ref: string | null;
+    metadata: Record<string, unknown>;
+}
+export interface RawSearchResult {
+    hits: RawSearchHit[];
+    next_cursor: string | null;
+}
+export type ConnectorEventStalenessStatus = 'healthy' | 'stale-but-warming' | 'warn' | 'unhealthy' | 'never_swept';
+//# sourceMappingURL=types.d.ts.map

package/dist/connectors/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/db-adapter/node-sqlite-adapter.d.ts

@@ -55,6 +55,12 @@ export declare class NodeSQLiteAdapter extends DatabaseAdapter {
     insertEmbedding(rowid: number, embedding: Float32Array | number[]): RunResult | null;
     getLastInsertRowid(): number;
     runMigrations(migrationsDir: string): void;
+    private recoverMemoryProvenanceMigration032;
+    private assertMigration032Complete;
+    private recoverConnectorEventScopeMigration034;
+    private assertMigration034Complete;
+    private tableColumns;
+    private indexExists;
     private migrateFromVssMemories;
 }
 export default NodeSQLiteAdapter;

package/dist/db-adapter/node-sqlite-adapter.js

@@ -18,6 +18,7 @@ const debug_logger_js_1 = require("../debug-logger.js");
 const embeddings_js_1 = require("../embeddings.js");
 const LEGACY_DB_PATH = path_1.default.join(os_1.default.homedir(), '.spinelift', 'memories.db');
 const DEFAULT_DB_PATH = path_1.default.join(os_1.default.homedir(), '.claude', 'mama-memory.db');
+const SQLITE_IDENTIFIER_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
 // Prefer better-sqlite3 (includes FTS5) over node:sqlite (lacks FTS5)
 let BetterSQLite3 = null;
 let DatabaseSync = null;
@@ -320,6 +321,16 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
             catch (err) {
                 this.exec('ROLLBACK');
                 const message = err instanceof Error ? err.message : String(err);
+                if (message.includes('duplicate column') && version === 32) {
+                    this.recoverMemoryProvenanceMigration032();
+                    (0, debug_logger_js_1.info)(`[node-sqlite-adapter] Migration ${file} recovered successfully`);
+                    continue;
+                }
+                if (message.includes('duplicate column') && version === 34) {
+                    this.recoverConnectorEventScopeMigration034();
+                    (0, debug_logger_js_1.info)(`[node-sqlite-adapter] Migration ${file} recovered successfully`);
+                    continue;
+                }
                 if (message.includes('duplicate column')) {
                     (0, debug_logger_js_1.warn)(`[node-sqlite-adapter] Migration ${file} skipped (duplicate column - already applied)`);
                     this.prepare('INSERT OR IGNORE INTO schema_version (version) VALUES (?)').run(version);
@@ -345,6 +356,126 @@ class NodeSQLiteAdapter extends base_adapter_js_1.DatabaseAdapter {
         }
         this.migrateFromVssMemories();
     }
+    recoverMemoryProvenanceMigration032() {
+        this.transaction(() => {
+            const expectedDecisionColumns = [
+                'agent_id',
+                'model_run_id',
+                'envelope_hash',
+                'gateway_call_id',
+                'source_refs_json',
+                'provenance_json',
+            ];
+            const decisionColumns = this.tableColumns('decisions');
+            for (const column of expectedDecisionColumns) {
+                if (!decisionColumns.has(column)) {
+                    this.exec(`ALTER TABLE decisions ADD COLUMN ${column} TEXT`);
+                    decisionColumns.add(column);
+                }
+            }
+            this.exec(`
+        CREATE INDEX IF NOT EXISTS idx_decisions_envelope_hash
+          ON decisions(envelope_hash)
+      `);
+            this.exec(`
+        CREATE INDEX IF NOT EXISTS idx_decisions_model_run_id
+          ON decisions(model_run_id)
+      `);
+            this.exec(`
+        CREATE INDEX IF NOT EXISTS idx_decisions_gateway_call_id
+          ON decisions(gateway_call_id)
+      `);
+            this.exec(`
+        CREATE INDEX IF NOT EXISTS idx_memory_events_memory_created
+          ON memory_events(memory_id, created_at DESC)
+      `);
+            this.assertMigration032Complete();
+            this.prepare('INSERT OR IGNORE INTO schema_version (version, description) VALUES (?, ?)').run(32, 'Add nullable memory provenance columns');
+        });
+    }
+    assertMigration032Complete() {
+        const decisionColumns = this.tableColumns('decisions');
+        for (const column of [
+            'agent_id',
+            'model_run_id',
+            'envelope_hash',
+            'gateway_call_id',
+            'source_refs_json',
+            'provenance_json',
+        ]) {
+            if (!decisionColumns.has(column)) {
+                throw new Error(`Migration 032 recovery failed: missing decisions.${column}`);
+            }
+        }
+        for (const indexName of [
+            'idx_decisions_envelope_hash',
+            'idx_decisions_model_run_id',
+            'idx_decisions_gateway_call_id',
+            'idx_memory_events_memory_created',
+        ]) {
+            if (!this.indexExists(indexName)) {
+                throw new Error(`Migration 032 recovery failed: missing index ${indexName}`);
+            }
+        }
+    }
+    recoverConnectorEventScopeMigration034() {
+        this.transaction(() => {
+            const expectedColumns = [
+                'source_cursor',
+                'tenant_id',
+                'project_id',
+                'memory_scope_kind',
+                'memory_scope_id',
+            ];
+            const columns = this.tableColumns('connector_event_index');
+            for (const column of expectedColumns) {
+                if (!columns.has(column)) {
+                    this.exec(`ALTER TABLE connector_event_index ADD COLUMN ${column} TEXT`);
+                    columns.add(column);
+                }
+            }
+            this.exec(`
+        CREATE INDEX IF NOT EXISTS idx_connector_event_scope
+          ON connector_event_index(tenant_id, project_id, memory_scope_kind, memory_scope_id)
+      `);
+            this.exec(`
+        CREATE INDEX IF NOT EXISTS idx_connector_event_source_cursor
+          ON connector_event_index(source_connector, source_cursor)
+      `);
+            this.assertMigration034Complete();
+            this.prepare('INSERT OR IGNORE INTO schema_version (version, description) VALUES (?, ?)').run(34, 'Add connector event scope columns');
+        });
+    }
+    assertMigration034Complete() {
+        const columns = this.tableColumns('connector_event_index');
+        for (const column of [
+            'source_cursor',
+            'tenant_id',
+            'project_id',
+            'memory_scope_kind',
+            'memory_scope_id',
+        ]) {
+            if (!columns.has(column)) {
+                throw new Error(`Migration 034 recovery failed: missing connector_event_index.${column}`);
+            }
+        }
+        for (const indexName of ['idx_connector_event_scope', 'idx_connector_event_source_cursor']) {
+            if (!this.indexExists(indexName)) {
+                throw new Error(`Migration 034 recovery failed: missing index ${indexName}`);
+            }
+        }
+    }
+    tableColumns(tableName) {
+        if (!SQLITE_IDENTIFIER_PATTERN.test(tableName)) {
+            throw new Error('Invalid SQLite table identifier');
+        }
+        const rows = this.prepare(`PRAGMA table_info(${tableName})`).all();
+        return new Set(rows.map((row) => row.name));
+    }
+    indexExists(indexName) {
+        const row = this.prepare("SELECT name FROM sqlite_master WHERE type='index' AND name = ?").get(indexName);
+        return row?.name === indexName;
+    }
     migrateFromVssMemories() {
         try {
             const vssTables = this.prepare(`SELECT name FROM sqlite_master WHERE name='vss_memories'`).all();

package/dist/db-manager.d.ts

@@ -57,6 +57,8 @@ export interface DecisionRecord {
     edges?: DecisionEdgeRow[];
     /** ISO 8601 date when the event actually occurred. Null if not set. */
     event_date?: string | null;
+    /** Source event timestamp in milliseconds when known. Null if not set. */
+    event_datetime?: number | null;
 }
 export interface OutcomeData {
     outcome?: string | null;
@@ -119,6 +121,7 @@ export interface SemanticEdges {
  * @returns SQLite database connection
  */
 export declare function initDB(): Promise<unknown>;
+export declare function assertTestProcessIsNotUsingRealDb(effectivePath?: string, effectivePathSource?: string): void;
 /**
  * Get database connection (singleton pattern)
  *
@@ -202,6 +205,14 @@ export interface DecisionInput {
     risks?: string | null;
     /** ISO 8601 date string for when the event actually occurred (e.g. "2023-01-15") */
     event_date?: string | null;
+    /** Source event timestamp in milliseconds when known. */
+    event_datetime?: number | null;
+    agent_id?: string | null;
+    model_run_id?: string | null;
+    envelope_hash?: string | null;
+    gateway_call_id?: string | null;
+    source_refs_json?: string | null;
+    provenance_json?: string | null;
 }
 /**
  * Insert decision with embedding
@@ -280,7 +291,7 @@ export declare function resetDBState(options?: {
 /**
  * Check if running in test mode
  *
- * Returns true if MAMA_TEST_MODE or VITEST env vars are set
+ * Returns true if MAMA_TEST_MODE, VITEST, or NODE_ENV=test env vars are set
  */
 export declare function isTestMode(): boolean;
 /**