@openanonymity/nanomem 0.1.0

package/src/types.js

@@ -0,0 +1,382 @@
+/**
+ * Shared type definitions for @openanonymity/nanomem.
+ *
+ * This file contains only JSDoc typedefs. It has no runtime code.
+ * Import types with: @import { TypeName } from './types.js'
+ */
+
+// ─── Bullet types ────────────────────────────────────────────────────────────
+
+/**
+ * @typedef {'working' | 'long_term' | 'history'} Tier
+ */
+
+/**
+ * @typedef {'active' | 'superseded' | 'expired' | 'uncertain'} Status
+ */
+
+/**
+ * @typedef {'user_statement' | 'llm_infer' | 'document' | 'document_infer' | 'assistant_summary' | 'inference' | 'system'} Source
+ */
+
+/**
+ * @typedef {'high' | 'medium' | 'low'} Confidence
+ */
+
+/**
+ * @typedef {object} Bullet
+ * @property {string} text
+ * @property {string | null} topic
+ * @property {string | null} updatedAt
+ * @property {string | null} expiresAt
+ * @property {string | null} reviewAt
+ * @property {Tier} tier
+ * @property {Status} status
+ * @property {Source | null} source
+ * @property {Confidence | null} confidence
+ * @property {boolean} explicitTier
+ * @property {boolean} explicitStatus
+ * @property {boolean} explicitSource
+ * @property {boolean} explicitConfidence
+ * @property {string} heading
+ * @property {string} section
+ * @property {number} lineIndex
+ */
+
+/**
+ * @typedef {object} EnsureBulletMetadataOptions
+ * @property {string} [defaultTopic]
+ * @property {Tier} [defaultTier]
+ * @property {Status} [defaultStatus]
+ * @property {Source} [defaultSource]
+ * @property {Confidence} [defaultConfidence]
+ * @property {string} [updatedAt]
+ */
+
+/**
+ * @typedef {object} CompactionResult
+ * @property {Bullet[]} working
+ * @property {Bullet[]} longTerm
+ * @property {Bullet[]} history
+ * @property {Bullet[]} active
+ * @property {Bullet[]} archive
+ */
+
+/**
+ * @typedef {object} CompactBulletsOptions
+ * @property {string} [today]
+ * @property {number} [maxActivePerTopic]
+ * @property {string} [defaultTopic]
+ */
+
+// ─── Storage types ───────────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} StorageMetadata
+ * @property {string} [oneLiner]
+ * @property {number} [itemCount]
+ * @property {string[]} [titles]
+ */
+
+/**
+ * @typedef {object} ExportRecord
+ * @property {string} path
+ * @property {string} [content]
+ * @property {string} [oneLiner]
+ * @property {number} [itemCount]
+ * @property {string[]} [titles]
+ * @property {string} [parentPath]
+ * @property {number} [createdAt]
+ * @property {number} [updatedAt]
+ */
+
+/**
+ * @typedef {object} SearchResult
+ * @property {string} path
+ * @property {string[]} lines
+ */
+
+/**
+ * @typedef {object} ListResult
+ * @property {string[]} files
+ * @property {string[]} dirs
+ */
+
+// ─── LLM types ──────────────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} ToolCallFunction
+ * @property {string} name
+ * @property {string} arguments
+ */
+
+/**
+ * @typedef {object} ToolCall
+ * @property {string} id
+ * @property {'function'} type
+ * @property {ToolCallFunction} function
+ */
+
+/**
+ * @typedef {object} ToolFunctionParameters
+ * @property {'object'} type
+ * @property {Record<string, { type: string; description?: string }>} properties
+ * @property {string[]} required
+ */
+
+/**
+ * @typedef {object} ToolFunctionDef
+ * @property {string} name
+ * @property {string} description
+ * @property {ToolFunctionParameters} parameters
+ */
+
+/**
+ * @typedef {object} ToolDefinition
+ * @property {'function'} type
+ * @property {ToolFunctionDef} function
+ */
+
+/**
+ * @typedef {object} LLMMessage
+ * @property {'system' | 'user' | 'assistant' | 'tool'} role
+ * @property {string | null} [content]
+ * @property {ToolCall[]} [tool_calls]
+ * @property {string} [tool_call_id]
+ */
+
+/**
+ * @typedef {object} ChatCompletionParams
+ * @property {string} model
+ * @property {LLMMessage[]} messages
+ * @property {ToolDefinition[]} [tools]
+ * @property {number} [max_tokens]
+ * @property {number} [temperature]
+ */
+
+/**
+ * @typedef {object} StreamChatCompletionParams
+ * @property {string} model
+ * @property {LLMMessage[]} messages
+ * @property {ToolDefinition[]} [tools]
+ * @property {number} [max_tokens]
+ * @property {number} [temperature]
+ * @property {(text: string) => void} [onDelta]
+ * @property {(text: string) => void} [onReasoning]
+ */
+
+/**
+ * @typedef {object} ChatCompletionResponse
+ * @property {string} content
+ * @property {ToolCall[]} tool_calls
+ * @property {string} [finish_reason]
+ * @property {{ prompt_tokens: number; completion_tokens: number } | null} usage
+ */
+
+/**
+ * @typedef {object} LLMClient
+ * @property {(params: ChatCompletionParams) => Promise<ChatCompletionResponse>} createChatCompletion
+ * @property {(params: StreamChatCompletionParams) => Promise<ChatCompletionResponse>} streamChatCompletion
+ */
+
+/**
+ * @typedef {object} LLMClientOptions
+ * @property {string} apiKey
+ * @property {string} [baseUrl]
+ * @property {Record<string, string>} [headers]
+ */
+
+// ─── Engine types ────────────────────────────────────────────────────────────
+
+/**
+ * @typedef {'init' | 'retrieval' | 'fallback' | 'tool_call' | 'reasoning' | 'loading' | 'complete'} ProgressStage
+ */
+
+/**
+ * @typedef {object} ProgressEvent
+ * @property {ProgressStage} stage
+ * @property {string} message
+ * @property {string} [tool]
+ * @property {Record<string, any>} [args]
+ * @property {string | Record<string, any>} [result]
+ * @property {string[]} [paths]
+ * @property {number} [iteration]
+ * @property {string} [path]
+ */
+
+/**
+ * @typedef {object} RetrievalResult
+ * @property {{ path: string; content: string }[]} files
+ * @property {string[]} paths
+ * @property {string | null} assembledContext
+ */
+
+/**
+ * @typedef {object} IngestOptions
+ * @property {string} [updatedAt]
+ * @property {'conversation' | 'document' | string} [mode] - Prompt set to use for extraction
+ * @property {'conversation' | 'document' | string} [extractionMode] - Alias for mode (deprecated, use mode)
+ * @property {string} [sessionTitle]
+ */
+
+/**
+ * @typedef {object} IngestResult
+ * @property {'processed' | 'skipped' | 'error'} status
+ * @property {number} writeCalls
+ * @property {Array<{path: string, before: string | null, after: string | null}>} [writes]
+ * @property {string} [error]
+ */
+
+/**
+ * @typedef {object} ToolCallLogEntry
+ * @property {string} name
+ * @property {Record<string, any>} args
+ * @property {string} result
+ * @property {string} toolCallId
+ */
+
+/**
+ * @typedef {object} ToolLoopResult
+ * @property {string} textResponse
+ * @property {{ name: string; arguments: Record<string, any> } | null} terminalToolResult
+ * @property {LLMMessage[]} messages
+ * @property {number} iterations
+ * @property {ToolCallLogEntry[]} toolCallLog
+ */
+
+/**
+ * @typedef {object} ToolLoopOptions
+ * @property {LLMClient} llmClient
+ * @property {string} model
+ * @property {ToolDefinition[]} tools
+ * @property {Record<string, ToolExecutor>} toolExecutors
+ * @property {LLMMessage[]} messages
+ * @property {string | null} [terminalTool]
+ * @property {number} [maxIterations]
+ * @property {number} [maxOutputTokens]
+ * @property {number} [temperature]
+ * @property {((name: string, args: Record<string, any>, result: string) => void) | null} [onToolCall]
+ * @property {((text: string, iteration: number) => void) | null} [onModelText]
+ * @property {((chunk: string, iteration: number) => void) | null} [onReasoning]
+ * @property {AbortSignal | null} [signal]
+ */
+
+/**
+ * @typedef {object} ExtractionExecutorHooks
+ * @property {(content: string, path: string) => string} [normalizeContent]
+ * @property {(existing: string | null, incoming: string, path: string) => string} [mergeWithExisting]
+ * @property {(path: string) => Promise<void>} [refreshIndex]
+ * @property {(path: string, before: string, after: string) => void} [onWrite]
+ */
+
+/**
+ * @typedef {(args: any) => Promise<string>} ToolExecutor
+ */
+
+/**
+ * @typedef {object} StorageBackend
+ * @property {() => Promise<void>} init
+ * @property {(path: string) => Promise<string | null>} read
+ * @property {(path: string, content: string) => Promise<void>} write
+ * @property {(path: string) => Promise<void>} delete
+ * @property {(path: string) => Promise<boolean>} exists
+ * @property {(query: string) => Promise<SearchResult[]>} search
+ * @property {(dirPath?: string) => Promise<ListResult>} ls
+ * @property {() => Promise<string | null>} getTree
+ * @property {() => Promise<void>} rebuildTree
+ * @property {() => Promise<ExportRecord[]>} exportAll
+ * @property {() => Promise<void>} clear
+ */
+
+/**
+ * @typedef {object} StorageFacade
+ * @property {(path: string) => Promise<string | null>} read
+ * @property {(path: string, content: string) => Promise<void>} write
+ * @property {(path: string) => Promise<void>} delete
+ * @property {(path: string) => Promise<boolean>} exists
+ * @property {(query: string) => Promise<SearchResult[]>} search
+ * @property {(dirPath?: string) => Promise<ListResult>} ls
+ * @property {() => Promise<string | null>} getTree
+ * @property {() => Promise<void>} rebuildTree
+ * @property {() => Promise<ExportRecord[]>} exportAll
+ * @property {() => Promise<void>} clear
+ */
+
+// ─── Memory bank config ─────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} MemoryBankLLMConfig
+ * @property {string} apiKey
+ * @property {string} [baseUrl]
+ * @property {string} [model]
+ * @property {'openai' | 'anthropic' | 'tinfoil' | 'custom' | string} [provider]
+ * @property {Record<string, string>} [headers]
+ */
+
+/**
+ * @typedef {object} MemoryBankConfig
+ * @property {MemoryBankLLMConfig} [llm]
+ * @property {LLMClient} [llmClient]
+ * @property {string} [model]
+ * @property {'ram' | 'filesystem' | 'indexeddb' | StorageBackend} [storage]
+ * @property {string} [storagePath]
+ * @property {(event: ProgressEvent) => void} [onProgress]
+ * @property {(event: ProgressEvent) => void} [onCompactProgress]
+ * @property {(name: string, args: Record<string, any>, result: string) => void} [onToolCall]
+ * @property {(text: string) => void} [onModelText]
+ */
+
+// ─── Import types ────────────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} Message
+ * @property {'user' | 'assistant'} role
+ * @property {string} content
+ */
+
+/**
+ * @typedef {object} ChatGptSession
+ * @property {string | null} title
+ * @property {Message[]} messages
+ * @property {string | null} updatedAt
+ */
+
+/**
+ * @typedef {object} SessionSummary
+ * @property {string} id
+ * @property {string} title
+ * @property {number | null} createdAt
+ * @property {number | null} updatedAt
+ * @property {number | null} messageCount
+ * @property {string | null} model
+ */
+
+/**
+ * @typedef {object} SessionWithConversation
+ * @property {SessionSummary} session
+ * @property {Message[]} conversation
+ */
+
+// ─── Bullet index types ──────────────────────────────────────────────────────
+
+/**
+ * @typedef {object} BulletItem
+ * @property {string} path
+ * @property {Bullet} bullet
+ * @property {number} fileUpdatedAt
+ */
+
+/**
+ * @typedef {object} MemoryBank
+ * @property {() => Promise<void>} init
+ * @property {(query: string, conversationText?: string) => Promise<RetrievalResult | null>} retrieve
+ * @property {(messages: Message[], options?: IngestOptions) => Promise<IngestResult>} ingest
+ * @property {() => Promise<{filesChanged: number, filesTotal: number} | undefined>} compact
+ * @property {StorageFacade} storage
+ * @property {() => Promise<string>} serialize
+ * @property {() => Promise<Uint8Array>} toZip
+ * @property {StorageBackend} _backend
+ * @property {import('./bullets/bulletIndex.js').MemoryBulletIndex} _bulletIndex
+ */
+
+export {};

package/src/utils/portability.js

@@ -0,0 +1,174 @@
+/**
+ * Portability utilities — convert memory state to/from portable formats.
+ *
+ * serialize(records)   → string      (line-delimited text, one file per block)
+ * deserialize(str)     → records     (inverse of serialize)
+ * toZip(records)       → Uint8Array  (valid ZIP archive, STORE mode, zero deps)
+ *
+ * `records` is the shape returned by backend.exportAll():
+ *   [{ path: string, content: string }, ...]
+ */
+/** @import { ExportRecord } from '../types.js' */
+
+// ─── Text serialization ───────────────────────────────────────────────────────
+
+const FILE_PREFIX = '--- FILE: ';
+
+/**
+ * Convert an array of {path, content} records into a single portable string.
+ *
+ * Format:
+ *   --- FILE: path/to/file.md
+ *   <file content>
+ *   --- FILE: other/file.md
+ *   <file content>
+ *
+ * Note: file content must not contain a line that starts with "--- FILE: ".
+ * This is extremely unlikely for memory markdown but worth being aware of.
+ * @param {ExportRecord[]} records
+ * @returns {string}
+ */
+export function serialize(records) {
+    return records
+        .filter(({ path }) => !path.endsWith('_tree.md'))
+        .map(({ path, content }) => `${FILE_PREFIX}${path}\n${content ?? ''}`)
+        .join('\n');
+}
+
+/**
+ * Reconstruct records from a serialized string produced by serialize().
+ * @param {string} str
+ * @returns {{ path: string, content: string }[]}
+ */
+export function deserialize(str) {
+    const records = [];
+    const lines = str.split('\n');
+    let currentPath = null;
+    let currentLines = [];
+
+    for (const line of lines) {
+        if (line.startsWith(FILE_PREFIX)) {
+            if (currentPath !== null) {
+                records.push({ path: currentPath, content: currentLines.join('\n') });
+            }
+            currentPath = line.slice(FILE_PREFIX.length);
+            currentLines = [];
+        } else {
+            if (currentPath !== null) currentLines.push(line);
+        }
+    }
+
+    if (currentPath !== null) {
+        records.push({ path: currentPath, content: currentLines.join('\n') });
+    }
+
+    return records;
+}
+
+// ─── ZIP archive ──────────────────────────────────────────────────────────────
+
+// CRC-32 lookup table (built once at module load)
+const CRC_TABLE = (() => {
+    const t = new Uint32Array(256);
+    for (let i = 0; i < 256; i++) {
+        let c = i;
+        for (let j = 0; j < 8; j++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1);
+        t[i] = c;
+    }
+    return t;
+})();
+
+function crc32(data) {
+    let crc = 0xffffffff;
+    for (let i = 0; i < data.length; i++) crc = CRC_TABLE[(crc ^ data[i]) & 0xff] ^ (crc >>> 8);
+    return (crc ^ 0xffffffff) >>> 0;
+}
+
+function u16(n) { return [(n & 0xff), (n >> 8) & 0xff]; }
+function u32(n) { return [(n & 0xff), (n >> 8) & 0xff, (n >> 16) & 0xff, (n >> 24) & 0xff]; }
+
+function concatU8(arrays) {
+    const total = arrays.reduce((s, a) => s + a.length, 0);
+    const out = new Uint8Array(total);
+    let off = 0;
+    for (const a of arrays) { out.set(a, off); off += a.length; }
+    return out;
+}
+
+/**
+ * Produce a valid ZIP archive (STORE, no compression) from an array of records.
+ * Works in Node.js ≥ 18 and modern browsers (uses TextEncoder + Uint8Array only).
+ *
+ * @param {ExportRecord[]} records
+ * @returns {Uint8Array}
+ */
+export function toZip(records) {
+    const enc = new TextEncoder();
+    const localParts = [];   // raw bytes for local file entries
+    const centralParts = []; // raw bytes for central directory entries
+    let localOffset = 0;
+    const filtered = records.filter(r => !r.path.endsWith('_tree.md'));
+
+    for (const { path, content } of filtered) {
+        const name = enc.encode(path);
+        const data = enc.encode(content ?? '');
+        const checksum = crc32(data);
+        const size = data.length;
+
+        // Local file header (30 bytes) + name + data
+        const localHeader = new Uint8Array([
+            0x50, 0x4b, 0x03, 0x04, // signature
+            0x14, 0x00,             // version needed: 2.0
+            0x00, 0x00,             // flags
+            0x00, 0x00,             // compression: STORE
+            0x00, 0x00,             // mod time
+            0x00, 0x00,             // mod date
+            ...u32(checksum),
+            ...u32(size),           // compressed size
+            ...u32(size),           // uncompressed size
+            ...u16(name.length),
+            0x00, 0x00,             // extra field length
+        ]);
+        localParts.push(localHeader, name, data);
+
+        // Central directory entry (46 bytes) + name
+        const cdEntry = new Uint8Array([
+            0x50, 0x4b, 0x01, 0x02, // signature
+            0x14, 0x00,             // version made by: 2.0
+            0x14, 0x00,             // version needed: 2.0
+            0x00, 0x00,             // flags
+            0x00, 0x00,             // compression: STORE
+            0x00, 0x00,             // mod time
+            0x00, 0x00,             // mod date
+            ...u32(checksum),
+            ...u32(size),           // compressed size
+            ...u32(size),           // uncompressed size
+            ...u16(name.length),
+            0x00, 0x00,             // extra field length
+            0x00, 0x00,             // file comment length
+            0x00, 0x00,             // disk number start
+            0x00, 0x00,             // internal attributes
+            0x00, 0x00, 0x00, 0x00, // external attributes
+            ...u32(localOffset),
+        ]);
+        centralParts.push(cdEntry, name);
+
+        localOffset += localHeader.length + name.length + data.length;
+    }
+
+    const centralDir = concatU8(centralParts);
+
+    // End of central directory record (22 bytes)
+    const eocd = new Uint8Array([
+        0x50, 0x4b, 0x05, 0x06, // signature
+        0x00, 0x00,             // disk number
+        0x00, 0x00,             // disk with central dir start
+        ...u16(filtered.length), // entries on this disk
+        ...u16(filtered.length), // total entries
+        ...u32(centralDir.length),
+        ...u32(localOffset),    // central dir offset
+        0x00, 0x00,             // comment length
+    ]);
+
+    return concatU8([...localParts, centralDir, eocd]);
+}

package/types/backends/BaseStorage.d.ts

@@ -0,0 +1,42 @@
+export class BaseStorage {
+    /** @returns {Promise<void>} */ init(): Promise<void>;
+    /** @returns {Promise<string | null>} */ _readRaw(_path: any): Promise<string | null>;
+    _writeRaw(_path: any, _content: any, _meta: any): Promise<void>;
+    /** @param {string} _path @returns {Promise<void>} */ delete(_path: string): Promise<void>;
+    /** @param {string} _path @returns {Promise<boolean>} */ exists(_path: string): Promise<boolean>;
+    /** @returns {Promise<void>} */ rebuildTree(): Promise<void>;
+    /** @returns {Promise<ExportRecord[]>} */ exportAll(): Promise<ExportRecord[]>;
+    /** @returns {Promise<void>} */ clear(): Promise<void>;
+    /**
+     * @param {string} path
+     * @returns {Promise<string | null>}
+     */
+    read(path: string): Promise<string | null>;
+    /**
+     * @param {string} path
+     * @param {string} content
+     * @returns {Promise<void>}
+     */
+    write(path: string, content: string): Promise<void>;
+    /** @returns {Promise<string | null>} */
+    getTree(): Promise<string | null>;
+    /**
+     * @param {string} query
+     * @returns {Promise<SearchResult[]>}
+     */
+    search(query: string): Promise<SearchResult[]>;
+    /**
+     * @param {string} [dirPath]
+     * @returns {Promise<ListResult>}
+     */
+    ls(dirPath?: string): Promise<ListResult>;
+    _isInternalPath(path: any): boolean;
+    /** Override for efficient path listing. Default uses exportAll(). */
+    _listAllPaths(): Promise<string[]>;
+    _parentPath(filePath: any): any;
+    /** Generate a one-line summary of file content for the index. */
+    _generateOneLiner(content: any): any;
+}
+import type { ExportRecord } from '../types.js';
+import type { SearchResult } from '../types.js';
+import type { ListResult } from '../types.js';

package/types/backends/filesystem.d.ts

@@ -0,0 +1,11 @@
+export class FileSystemStorage extends BaseStorage {
+    constructor(rootDir: any);
+    _root: any;
+    _initialized: boolean;
+    _writeRaw(path: any, content: any, _meta?: {}): Promise<void>;
+    _listAllPaths(): Promise<any>;
+    _resolve(path: any): string;
+    _ensureDir(dirPath: any): Promise<void>;
+    _walkFiles(dir?: string): any;
+}
+import { BaseStorage } from './BaseStorage.js';

package/types/backends/indexeddb.d.ts

@@ -0,0 +1,12 @@
+export class IndexedDBStorage extends BaseStorage {
+    /** @type {IDBDatabase | null} */
+    db: IDBDatabase | null;
+    /** @type {Promise<IDBDatabase> | null} */
+    _initPromise: Promise<IDBDatabase> | null;
+    _bootstrap(): Promise<void>;
+    _readRaw(path: any): Promise<any>;
+    _writeRaw(path: any, content: any, meta?: {}): Promise<void>;
+    _get(path: any): Promise<any>;
+    _getAll(): Promise<any>;
+}
+import { BaseStorage } from './BaseStorage.js';

package/types/backends/ram.d.ts

@@ -0,0 +1,8 @@
+export class InMemoryStorage extends BaseStorage {
+    _files: Map<any, any>;
+    _initialized: boolean;
+    _readRaw(path: any): Promise<any>;
+    _writeRaw(path: any, content: any, meta?: {}): Promise<void>;
+    _listAllPaths(): Promise<any[]>;
+}
+import { BaseStorage } from './BaseStorage.js';

package/types/backends/schema.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Memory index generation and bootstrap.
+ */
+/** @import { ExportRecord } from '../types.js' */
+export function buildBootstrapIndex(): string;
+export function createBootstrapRecords(now?: number): {
+    path: string;
+    content: string;
+    oneLiner: string;
+    parentPath: string;
+    createdAt: number;
+    updatedAt: number;
+}[];
+export function buildTree(files: any): string;

package/types/bullets/bulletIndex.d.ts

@@ -0,0 +1,47 @@
+export class MemoryBulletIndex {
+    /**
+     * @param {StorageBackend} backend
+     */
+    constructor(backend: StorageBackend);
+    _backend: StorageBackend;
+    _initialized: boolean;
+    _initPromise: Promise<void> | null;
+    _pathToBullets: Map<any, any>;
+    _pathToUpdatedAt: Map<any, any>;
+    /** @returns {Promise<void>} */
+    init(): Promise<void>;
+    /** @returns {Promise<void>} */
+    rebuild(): Promise<void>;
+    _rebuild(): Promise<void>;
+    _parseForIndex(path: any, content: any): Bullet[] | {
+        text: string;
+        topic: any;
+        updatedAt: string;
+        expiresAt: null;
+        reviewAt: null;
+        tier: string;
+        status: string;
+        source: null;
+        confidence: null;
+        explicitTier: boolean;
+        explicitStatus: boolean;
+        explicitSource: boolean;
+        explicitConfidence: boolean;
+        heading: string;
+        section: string;
+        lineIndex: number;
+    }[];
+    /**
+     * @param {string} path
+     * @returns {Promise<void>}
+     */
+    refreshPath(path: string): Promise<void>;
+    /**
+     * @param {string[]} paths
+     * @returns {BulletItem[]}
+     */
+    getBulletsForPaths(paths: string[]): BulletItem[];
+}
+import type { StorageBackend } from '../types.js';
+import type { Bullet } from '../types.js';
+import type { BulletItem } from '../types.js';