@equationalapplications/expo-llm-wiki 0.0.0-development

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Equational Applications
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,259 @@
+# expo-llm-wiki
+
+Offline-first, SQLite-backed memory for LLM apps built with Expo. Handles FTS5 search, episodic event logging, background fact extraction, and memory healing — bring your own LLM.
+
+## Key Principles
+
+- **Bring Your Own Inference (BYOI):** Provide one `generateText` function. The package owns prompt construction, JSON parsing, and database writes.
+- **Namespace Safe:** All tables are prefixed (default: `llm_wiki_`) — no collisions with your existing database.
+- **Multi-Entity:** Multiple independent "brains" in one database via `entityId`.
+- **Offline First:** Reads are fully local via SQLite FTS5, typically under 50ms.
+- **Full Unicode Support:** UTF-8 and UTF-16 (including surrogate pairs for emoji) are fully supported. Chunks are split safely at sentence boundaries; surrogate pairs are never fragmented.
+
+## How It Works
+
+```mermaid
+flowchart LR
+    subgraph API
+        direction TB
+        write["write(event)"]
+        ingest["ingestDocument()"]
+        librarian["runLibrarian()"]
+        heal["runHeal()"]
+    end
+
+    subgraph SQLite
+        direction TB
+        events[(events)]
+        entries[("entries\n(facts)")]
+        tasks[(tasks)]
+    end
+
+    LLM["LLMProvider\n.generateText()"]
+
+    read["read(entityId, query)"]
+    FTS5(["FTS5 search"])
+    Bundle(["MemoryBundle\nfacts · tasks · events"])
+
+    write --> events
+    events -. "≥ threshold" .-> librarian
+    librarian --> LLM
+    heal --> LLM
+    ingest --> LLM
+    LLM --> entries
+    LLM --> tasks
+
+    read --> FTS5
+    FTS5 --> entries
+    entries --> Bundle
+    tasks --> Bundle
+    events --> Bundle
+```
+
+## Installation
+
+In your Expo project:
+
+```bash
+npx expo install expo-sqlite
+npm install expo-llm-wiki
+```
+
+Use `npx expo install` for `expo-sqlite` so Expo's version resolver picks the correct native build for your SDK version.
+
+## Setup
+
+```typescript
+import { createWiki } from 'expo-llm-wiki';
+import * as SQLite from 'expo-sqlite';
+
+const db = await SQLite.openDatabaseAsync('my-app.db');
+
+const wiki = createWiki(db, {
+  llmProvider: {
+    generateText: async ({ systemPrompt, userPrompt }) => {
+      // Connect to OpenAI, Gemini, a local model, etc.
+      // Must return a raw string (JSON, optionally in a markdown code fence).
+      const response = await openai.chat.completions.create({
+        model: 'gpt-4o-mini',
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+      });
+      return response.choices[0].message.content ?? '{}';
+    },
+  },
+  config: {
+    tablePrefix: 'llm_wiki_',       // optional, default: 'llm_wiki_'
+    maxFtsResults: 10,              // optional, default: 10
+    autoLibrarianThreshold: 20,     // optional, default: 20
+    maxChunkLength: 6000,           // optional, default: 6000 (char count, not bytes)
+  },
+});
+
+// Create tables and FTS5 indexes (call once on app startup)
+await wiki.setup();
+```
+
+## Core API
+
+### Read
+
+FTS5 full-text search over facts, plus open tasks and recent events:
+
+```typescript
+const { facts, tasks, events } = await wiki.read('entity-123', 'weekend plans');
+// facts: WikiFact[]   — matched by FTS5, ranked by confidence + access count
+// tasks: WikiTask[]   — pending and in-progress only
+// events: WikiEvent[] — 10 most recent, ascending
+```
+
+Pass an empty string to skip FTS and return the most recently updated facts.
+
+### Write
+
+Log an episodic event. Automatically triggers the librarian pass once enough events accumulate:
+
+```typescript
+await wiki.write('entity-123', {
+  event_type: 'observation',
+  summary: 'User mentioned they love hiking on weekends.',
+});
+// event_type: 'observation' | 'decision' | 'action' | 'outcome'
+```
+
+### Ingest Document
+
+Extract facts from a document (chunked internally). Idempotent — re-calling with the same `sourceRef` replaces the prior extraction. Documents are automatically chunked at sentence boundaries; if a sentence exceeds `maxChunkLength`, it is hard-split.
+
+```typescript
+const result = await wiki.ingestDocument('entity-123', {
+  // sourceRef is normalized: only [A-Za-z0-9._\- ] are kept, all other characters
+  // (including `/`) are stripped. Use underscores or dots as path separators to
+  // avoid accidental collisions (e.g. 'docs_preferences.md' not 'docs/preferences.md').
+  sourceRef: 'preferences.md',        // stable identifier
+  sourceHash: sha256(content),        // for change detection
+  documentChunk: content,
+  maxChunkLength: 6000,               // optional, character count
+});
+// result: { truncated: boolean; chunks: number }
+// truncated: true if at least one hard-split was required (no sentence boundary)
+// chunks: number of LLM calls made
+```
+
+### Background Maintenance
+
+```typescript
+// Consolidate recent events into durable facts (auto-triggered by write, or call manually)
+await wiki.runLibrarian('entity-123');
+
+// Resolve contradictions, downgrade stale claims, remove obsolete facts
+await wiki.runHeal('entity-123');
+```
+
+### Forget
+
+```typescript
+const result = await wiki.forget('entity-123', { entryId: 'fact_abc' });    // single fact
+// result: { deleted: { entries: number; tasks: number } }
+
+await wiki.forget('entity-123', { taskId: 'task_xyz' });     // single task
+// sourceRef is normalized the same way as in ingestDocument (slashes stripped)
+await wiki.forget('entity-123', { sourceRef: 'x.md' }); // all facts from a document
+await wiki.forget('entity-123', { clearAll: true });          // wipe entity
+```
+
+Throws `Error` if `sourceRef` or `sourceHash` is provided but invalid. Soft-deletes are idempotent — calling again with the same parameters returns `{ deleted: { entries: 0; tasks: 0 } }`.
+
+---
+
+## React / Expo Component API
+
+Import from `expo-llm-wiki/react`. This entry point is separate so non-React consumers do not transitively import React.
+
+### Provider
+
+Wrap once at app root (or any subtree that needs memory access):
+
+```typescript
+import { WikiProvider } from 'expo-llm-wiki/react';
+import { createWiki } from 'expo-llm-wiki';
+
+const wiki = createWiki(db, { llmProvider });
+
+export default function App() {
+  return (
+    <WikiProvider wiki={wiki}>
+      <YourApp />
+    </WikiProvider>
+  );
+}
+```
+
+### `useMemoryRead(entityId, query)`
+
+Reactive read. Fetches on mount and whenever `entityId` or `query` changes. In-flight results always land before a queued re-fetch starts — results are never silently discarded.
+
+```typescript
+const { data, isPending, error, refetch } = useMemoryRead('entity-123', 'weekend plans');
+// data: MemoryBundle | null
+```
+
+### `useWikiWrite()`
+
+```typescript
+const { execute, isPending, error } = useWikiWrite();
+
+await execute('entity-123', {
+  event_type: 'observation',
+  summary: 'User mentioned they love hiking.',
+});
+```
+
+### `useWikiMaintenance()`
+
+Shared `isPending` — true if either operation is in-flight:
+
+```typescript
+const { runLibrarian, runHeal, isPending, error } = useWikiMaintenance();
+
+await runLibrarian('entity-123');
+await runHeal('entity-123');
+```
+
+### `useWikiIngest()`
+
+```typescript
+const { execute, lastResult, isPending, error } = useWikiIngest();
+// lastResult: { truncated: boolean; chunks: number } | null
+
+const result = await execute('entity-123', {
+  sourceRef: 'preferences.md',  // slashes are stripped by normalizeSourceRef
+  sourceHash: sha256(content),
+  documentChunk: content,
+});
+// result.truncated — true if any hard-splits were required
+// result.chunks   — number of LLM calls made
+```
+
+### `useWikiForget()`
+
+```typescript
+const { execute, lastResult, isPending, error } = useWikiForget();
+// lastResult: { deleted: { entries: number; tasks: number } } | null
+
+const result = await execute('entity-123', { entryId: 'fact_abc' });
+// result.deleted.entries — rows soft-deleted
+```
+
+All mutation hooks follow the same pattern (`TResult` is specific per hook):
+
+```typescript
+{
+  execute: (...args) => Promise<TResult>;
+  lastResult: TResult | null;  // result of the last successful call; null before first call or after an error
+  isPending: boolean;
+  error: Error | null;         // cleared on the next execute call
+}
+```

package/dist/WikiMemory-B-yFw9Dc.d.mts

@@ -0,0 +1,118 @@
+import * as SQLite from 'expo-sqlite';
+
+interface WikiConfig {
+    tablePrefix?: string;
+    maxFtsResults?: number;
+    pruneEventsAfter?: number;
+    autoLibrarianThreshold?: number;
+    autoHealThreshold?: number;
+    orphanAfterDays?: number | null;
+    staleInferredAfterDays?: number | null;
+    maxChunkBytes?: number;
+}
+interface WikiFact {
+    id: string;
+    entity_id: string;
+    title: string;
+    body: string;
+    tags: string[];
+    confidence: 'certain' | 'inferred' | 'tentative';
+    source_type: 'user_stated' | 'agent_inferred' | 'user_confirmed' | 'user_document';
+    source_hash: string | null;
+    source_ref: string | null;
+    created_at: number;
+    updated_at: number;
+    last_accessed_at: number | null;
+    access_count: number;
+    deleted_at: number | null;
+}
+interface WikiTask {
+    id: string;
+    entity_id: string;
+    description: string;
+    status: 'pending' | 'in_progress' | 'done' | 'abandoned';
+    priority: number;
+    created_at: number;
+    updated_at: number;
+    resolved_at: number | null;
+    deleted_at: number | null;
+}
+interface WikiEvent {
+    id: string;
+    entity_id: string;
+    event_type: 'observation' | 'decision' | 'action' | 'outcome';
+    summary: string;
+    related_entry_id?: string | null;
+    created_at: number;
+}
+interface WikiCheckpoint {
+    entity_id: string;
+    heal_checkpoint: number;
+    memory_checkpoint: number;
+}
+interface ExtractedFact {
+    title: string;
+    body: string;
+    tags: string[];
+    confidence: 'certain' | 'inferred' | 'tentative';
+}
+interface ExtractedTask {
+    description: string;
+    priority: number;
+}
+interface LLMProvider {
+    /**
+     * Generates text using the developer's LLM of choice.
+     * Expected to return the raw text response (typically a JSON string).
+     */
+    generateText: (params: {
+        systemPrompt: string;
+        userPrompt: string;
+    }) => Promise<string>;
+}
+interface WikiOptions {
+    config?: WikiConfig;
+    llmProvider: LLMProvider;
+}
+interface MemoryBundle {
+    facts: WikiFact[];
+    tasks: WikiTask[];
+    events: WikiEvent[];
+}
+
+declare class WikiMemory {
+    private db;
+    private prefix;
+    private options;
+    constructor(db: SQLite.SQLiteDatabase, options: WikiOptions);
+    setup(): Promise<void>;
+    private formatSearchQuery;
+    read(entityId: string, query: string): Promise<MemoryBundle>;
+    write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
+    private runLibrarianThenMaybeHeal;
+    runLibrarian(entityId: string): Promise<void>;
+    runHeal(entityId: string): Promise<void>;
+    forget(entityId: string, params: {
+        entryId?: string;
+        taskId?: string;
+        sourceRef?: string;
+        sourceHash?: string;
+        clearAll?: boolean;
+    }): Promise<{
+        deleted: {
+            entries: number;
+            tasks: number;
+        };
+    }>;
+    ingestDocument(entityId: string, params: {
+        sourceRef: string;
+        sourceHash: string;
+        documentChunk: string;
+        maxChunkBytes?: number;
+    }): Promise<{
+        truncated: boolean;
+        chunks: number;
+    }>;
+}
+
+export { type ExtractedFact as E, type LLMProvider as L, type MemoryBundle as M, type WikiOptions as W, WikiMemory as a, type ExtractedTask as b, type WikiCheckpoint as c, type WikiConfig as d, type WikiEvent as e, type WikiFact as f, type WikiTask as g };

package/dist/WikiMemory-B-yFw9Dc.d.ts

@@ -0,0 +1,118 @@
+import * as SQLite from 'expo-sqlite';
+
+interface WikiConfig {
+    tablePrefix?: string;
+    maxFtsResults?: number;
+    pruneEventsAfter?: number;
+    autoLibrarianThreshold?: number;
+    autoHealThreshold?: number;
+    orphanAfterDays?: number | null;
+    staleInferredAfterDays?: number | null;
+    maxChunkBytes?: number;
+}
+interface WikiFact {
+    id: string;
+    entity_id: string;
+    title: string;
+    body: string;
+    tags: string[];
+    confidence: 'certain' | 'inferred' | 'tentative';
+    source_type: 'user_stated' | 'agent_inferred' | 'user_confirmed' | 'user_document';
+    source_hash: string | null;
+    source_ref: string | null;
+    created_at: number;
+    updated_at: number;
+    last_accessed_at: number | null;
+    access_count: number;
+    deleted_at: number | null;
+}
+interface WikiTask {
+    id: string;
+    entity_id: string;
+    description: string;
+    status: 'pending' | 'in_progress' | 'done' | 'abandoned';
+    priority: number;
+    created_at: number;
+    updated_at: number;
+    resolved_at: number | null;
+    deleted_at: number | null;
+}
+interface WikiEvent {
+    id: string;
+    entity_id: string;
+    event_type: 'observation' | 'decision' | 'action' | 'outcome';
+    summary: string;
+    related_entry_id?: string | null;
+    created_at: number;
+}
+interface WikiCheckpoint {
+    entity_id: string;
+    heal_checkpoint: number;
+    memory_checkpoint: number;
+}
+interface ExtractedFact {
+    title: string;
+    body: string;
+    tags: string[];
+    confidence: 'certain' | 'inferred' | 'tentative';
+}
+interface ExtractedTask {
+    description: string;
+    priority: number;
+}
+interface LLMProvider {
+    /**
+     * Generates text using the developer's LLM of choice.
+     * Expected to return the raw text response (typically a JSON string).
+     */
+    generateText: (params: {
+        systemPrompt: string;
+        userPrompt: string;
+    }) => Promise<string>;
+}
+interface WikiOptions {
+    config?: WikiConfig;
+    llmProvider: LLMProvider;
+}
+interface MemoryBundle {
+    facts: WikiFact[];
+    tasks: WikiTask[];
+    events: WikiEvent[];
+}
+
+declare class WikiMemory {
+    private db;
+    private prefix;
+    private options;
+    constructor(db: SQLite.SQLiteDatabase, options: WikiOptions);
+    setup(): Promise<void>;
+    private formatSearchQuery;
+    read(entityId: string, query: string): Promise<MemoryBundle>;
+    write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
+    private runLibrarianThenMaybeHeal;
+    runLibrarian(entityId: string): Promise<void>;
+    runHeal(entityId: string): Promise<void>;
+    forget(entityId: string, params: {
+        entryId?: string;
+        taskId?: string;
+        sourceRef?: string;
+        sourceHash?: string;
+        clearAll?: boolean;
+    }): Promise<{
+        deleted: {
+            entries: number;
+            tasks: number;
+        };
+    }>;
+    ingestDocument(entityId: string, params: {
+        sourceRef: string;
+        sourceHash: string;
+        documentChunk: string;
+        maxChunkBytes?: number;
+    }): Promise<{
+        truncated: boolean;
+        chunks: number;
+    }>;
+}
+
+export { type ExtractedFact as E, type LLMProvider as L, type MemoryBundle as M, type WikiOptions as W, WikiMemory as a, type ExtractedTask as b, type WikiCheckpoint as c, type WikiConfig as d, type WikiEvent as e, type WikiFact as f, type WikiTask as g };

package/dist/WikiMemory-BI2Aizwv.d.mts

@@ -0,0 +1,121 @@
+import * as SQLite from 'expo-sqlite';
+
+interface WikiConfig {
+    tablePrefix?: string;
+    maxFtsResults?: number;
+    pruneEventsAfter?: number;
+    autoLibrarianThreshold?: number;
+    autoHealThreshold?: number;
+    orphanAfterDays?: number | null;
+    staleInferredAfterDays?: number | null;
+    maxChunkLength?: number;
+}
+interface WikiFact {
+    id: string;
+    entity_id: string;
+    title: string;
+    body: string;
+    tags: string[];
+    confidence: 'certain' | 'inferred' | 'tentative';
+    source_type: 'user_stated' | 'agent_inferred' | 'user_confirmed' | 'user_document';
+    source_hash: string | null;
+    source_ref: string | null;
+    created_at: number;
+    updated_at: number;
+    last_accessed_at: number | null;
+    access_count: number;
+    deleted_at: number | null;
+}
+interface WikiTask {
+    id: string;
+    entity_id: string;
+    description: string;
+    status: 'pending' | 'in_progress' | 'done' | 'abandoned';
+    priority: number;
+    created_at: number;
+    updated_at: number;
+    resolved_at: number | null;
+    deleted_at: number | null;
+}
+interface WikiEvent {
+    id: string;
+    entity_id: string;
+    event_type: 'observation' | 'decision' | 'action' | 'outcome';
+    summary: string;
+    related_entry_id?: string | null;
+    created_at: number;
+}
+interface WikiCheckpoint {
+    entity_id: string;
+    heal_checkpoint: number;
+    memory_checkpoint: number;
+}
+interface ExtractedFact {
+    title: string;
+    body: string;
+    tags: string[];
+    confidence: 'certain' | 'inferred' | 'tentative';
+}
+interface ExtractedTask {
+    description: string;
+    priority: number;
+}
+interface LLMProvider {
+    /**
+     * Generates text using the developer's LLM of choice.
+     * Expected to return the raw text response (typically a JSON string).
+     */
+    generateText: (params: {
+        systemPrompt: string;
+        userPrompt: string;
+    }) => Promise<string>;
+}
+interface WikiOptions {
+    config?: WikiConfig;
+    llmProvider: LLMProvider;
+}
+interface MemoryBundle {
+    facts: WikiFact[];
+    tasks: WikiTask[];
+    events: WikiEvent[];
+}
+
+declare class WikiMemory {
+    private db;
+    private prefix;
+    private options;
+    private activeMaintenanceJobs;
+    constructor(db: SQLite.SQLiteDatabase, options: WikiOptions);
+    setup(): Promise<void>;
+    private formatSearchQuery;
+    read(entityId: string, query: string): Promise<MemoryBundle>;
+    write(entityId: string, event: Omit<WikiEvent, 'id' | 'entity_id' | 'created_at'>): Promise<void>;
+    private runLibrarianThenMaybeHeal;
+    private _doRunLibrarian;
+    private _doRunHeal;
+    runLibrarian(entityId: string): Promise<void>;
+    runHeal(entityId: string): Promise<void>;
+    forget(entityId: string, params: {
+        entryId?: string;
+        taskId?: string;
+        sourceRef?: string;
+        sourceHash?: string;
+        clearAll?: boolean;
+    }): Promise<{
+        deleted: {
+            entries: number;
+            tasks: number;
+        };
+    }>;
+    ingestDocument(entityId: string, params: {
+        sourceRef: string;
+        sourceHash: string;
+        documentChunk: string;
+        maxChunkLength?: number;
+    }): Promise<{
+        truncated: boolean;
+        chunks: number;
+    }>;
+}
+
+export { type ExtractedFact as E, type LLMProvider as L, type MemoryBundle as M, type WikiOptions as W, WikiMemory as a, type ExtractedTask as b, type WikiCheckpoint as c, type WikiConfig as d, type WikiEvent as e, type WikiFact as f, type WikiTask as g };