@equationalapplications/expo-llm-wiki 2.0.2 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/{WikiMemory-B54Gaa-K.d.mts → WikiMemory-CjlQ68X0.d.mts} +26 -3
- package/dist/{WikiMemory-B54Gaa-K.d.ts → WikiMemory-CjlQ68X0.d.ts} +26 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +88 -12
- package/dist/index.mjs +88 -12
- package/dist/react/index.d.mts +1 -1
- package/dist/react/index.d.ts +1 -1
- package/package.json +7 -1
package/README.md
CHANGED
|
@@ -13,6 +13,7 @@ Offline-first, SQLite-backed memory for LLM apps built with Expo. Handles FTS5 s
|
|
|
13
13
|
- **Namespace Safe:** All tables are prefixed (default: `llm_wiki_`) — no collisions with your existing database.
|
|
14
14
|
- **Multi-Entity:** Multiple independent "brains" in one database via `entityId`.
|
|
15
15
|
- **Offline First:** Reads are fully local via SQLite FTS5, typically under 50ms.
|
|
16
|
+
- **Morphological Matching:** Porter stemming enables recall across word forms — queries for `running` match facts about `run`, `runs`, etc., without manual synonym configuration.
|
|
16
17
|
- **Full Unicode Support:** UTF-8 and UTF-16 (including surrogate pairs for emoji) are fully supported. Chunks are split safely at sentence boundaries; surrogate pairs are never fragmented.
|
|
17
18
|
|
|
18
19
|
## How It Works
|
|
@@ -94,6 +95,8 @@ const wiki = createWiki(db, {
|
|
|
94
95
|
maxFtsResults: 10, // optional, default: 10
|
|
95
96
|
autoLibrarianThreshold: 20, // optional, default: 20
|
|
96
97
|
maxChunkLength: 6000, // optional, default: 6000 (char count, not bytes)
|
|
98
|
+
chunkOverlap: 400, // optional, default: 400 (overlap between chunks in characters)
|
|
99
|
+
chunkConcurrency: 1, // optional, default: 1 (parallel LLM calls per ingestDocument)
|
|
97
100
|
},
|
|
98
101
|
});
|
|
99
102
|
|
|
@@ -142,6 +145,7 @@ const result = await wiki.ingestDocument('entity-123', {
|
|
|
142
145
|
documentChunk: content,
|
|
143
146
|
maxChunkLength: 12000, // optional, character count
|
|
144
147
|
chunkOverlap: 400, // optional, overlap in characters
|
|
148
|
+
chunkConcurrency: 1, // optional, parallel LLM calls per ingest (default: 1)
|
|
145
149
|
});
|
|
146
150
|
// result: { truncated: boolean; chunks: number }
|
|
147
151
|
// truncated: true if at least one hard-split was required (no sentence boundary)
|
|
@@ -4,6 +4,7 @@ interface WikiConfig {
|
|
|
4
4
|
tablePrefix?: string;
|
|
5
5
|
maxFtsResults?: number;
|
|
6
6
|
pruneEventsAfter?: number;
|
|
7
|
+
pruneRetainSoftDeletedFor?: number;
|
|
7
8
|
autoLibrarianThreshold?: number;
|
|
8
9
|
autoHealThreshold?: number;
|
|
9
10
|
orphanAfterDays?: number | null;
|
|
@@ -11,6 +12,15 @@ interface WikiConfig {
|
|
|
11
12
|
maxChunkLength?: number;
|
|
12
13
|
chunkOverlap?: number;
|
|
13
14
|
chunkConcurrency?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Static caller-supplied synonym expansions applied at query time.
|
|
17
|
+
* Keys must match the same normalization pipeline used by query formatting:
|
|
18
|
+
* the query is lowercased, stripped to `[a-z0-9 ]`, split into tokens, and
|
|
19
|
+
* only tokens with length >= 3 are considered for synonym lookup.
|
|
20
|
+
* Values are appended to the FTS5 query token list (multi-word values are
|
|
21
|
+
* split into tokens), then deduped and sliced to 12.
|
|
22
|
+
*/
|
|
23
|
+
synonymMap?: Record<string, string[]>;
|
|
14
24
|
}
|
|
15
25
|
interface WikiFact {
|
|
16
26
|
id: string;
|
|
@@ -92,15 +102,28 @@ interface FormattedMemoryDump {
|
|
|
92
102
|
content: string;
|
|
93
103
|
}>;
|
|
94
104
|
}
|
|
105
|
+
interface FormatContextOptions {
|
|
106
|
+
format?: 'markdown' | 'plain';
|
|
107
|
+
maxFacts?: number;
|
|
108
|
+
maxTasks?: number;
|
|
109
|
+
maxEvents?: number;
|
|
110
|
+
includeConfidence?: boolean;
|
|
111
|
+
includeTags?: boolean;
|
|
112
|
+
factWeights?: {
|
|
113
|
+
confidence?: number;
|
|
114
|
+
accessCount?: number;
|
|
115
|
+
recency?: number;
|
|
116
|
+
};
|
|
117
|
+
}
|
|
95
118
|
interface EntityStatus {
|
|
96
119
|
ingesting: boolean;
|
|
97
120
|
librarian: boolean;
|
|
98
121
|
heal: boolean;
|
|
99
122
|
}
|
|
100
123
|
declare class WikiBusyError extends Error {
|
|
101
|
-
readonly operation: 'ingest' | 'librarian' | 'heal';
|
|
124
|
+
readonly operation: 'ingest' | 'librarian' | 'heal' | 'prune';
|
|
102
125
|
readonly entityId: string;
|
|
103
|
-
constructor(operation: 'ingest' | 'librarian' | 'heal', entityId: string);
|
|
126
|
+
constructor(operation: 'ingest' | 'librarian' | 'heal' | 'prune', entityId: string);
|
|
104
127
|
}
|
|
105
128
|
|
|
106
129
|
declare class WikiMemory {
|
|
@@ -154,4 +177,4 @@ declare class WikiMemory {
|
|
|
154
177
|
}>;
|
|
155
178
|
}
|
|
156
179
|
|
|
157
|
-
export { type EntityStatus as E, type FormattedMemoryDump as F, type LLMProvider as L, type MemoryDump as M, type WikiOptions as W, WikiMemory as a, type ExtractedFact as b, type ExtractedTask as c, type
|
|
180
|
+
export { type EntityStatus as E, type FormattedMemoryDump as F, type LLMProvider as L, type MemoryDump as M, type WikiOptions as W, WikiMemory as a, type ExtractedFact as b, type ExtractedTask as c, type FormatContextOptions as d, type MemoryBundle as e, WikiBusyError as f, type WikiCheckpoint as g, type WikiConfig as h, type WikiEvent as i, type WikiFact as j, type WikiTask as k };
|
|
@@ -4,6 +4,7 @@ interface WikiConfig {
|
|
|
4
4
|
tablePrefix?: string;
|
|
5
5
|
maxFtsResults?: number;
|
|
6
6
|
pruneEventsAfter?: number;
|
|
7
|
+
pruneRetainSoftDeletedFor?: number;
|
|
7
8
|
autoLibrarianThreshold?: number;
|
|
8
9
|
autoHealThreshold?: number;
|
|
9
10
|
orphanAfterDays?: number | null;
|
|
@@ -11,6 +12,15 @@ interface WikiConfig {
|
|
|
11
12
|
maxChunkLength?: number;
|
|
12
13
|
chunkOverlap?: number;
|
|
13
14
|
chunkConcurrency?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Static caller-supplied synonym expansions applied at query time.
|
|
17
|
+
* Keys must match the same normalization pipeline used by query formatting:
|
|
18
|
+
* the query is lowercased, stripped to `[a-z0-9 ]`, split into tokens, and
|
|
19
|
+
* only tokens with length >= 3 are considered for synonym lookup.
|
|
20
|
+
* Values are appended to the FTS5 query token list (multi-word values are
|
|
21
|
+
* split into tokens), then deduped and sliced to 12.
|
|
22
|
+
*/
|
|
23
|
+
synonymMap?: Record<string, string[]>;
|
|
14
24
|
}
|
|
15
25
|
interface WikiFact {
|
|
16
26
|
id: string;
|
|
@@ -92,15 +102,28 @@ interface FormattedMemoryDump {
|
|
|
92
102
|
content: string;
|
|
93
103
|
}>;
|
|
94
104
|
}
|
|
105
|
+
interface FormatContextOptions {
|
|
106
|
+
format?: 'markdown' | 'plain';
|
|
107
|
+
maxFacts?: number;
|
|
108
|
+
maxTasks?: number;
|
|
109
|
+
maxEvents?: number;
|
|
110
|
+
includeConfidence?: boolean;
|
|
111
|
+
includeTags?: boolean;
|
|
112
|
+
factWeights?: {
|
|
113
|
+
confidence?: number;
|
|
114
|
+
accessCount?: number;
|
|
115
|
+
recency?: number;
|
|
116
|
+
};
|
|
117
|
+
}
|
|
95
118
|
interface EntityStatus {
|
|
96
119
|
ingesting: boolean;
|
|
97
120
|
librarian: boolean;
|
|
98
121
|
heal: boolean;
|
|
99
122
|
}
|
|
100
123
|
declare class WikiBusyError extends Error {
|
|
101
|
-
readonly operation: 'ingest' | 'librarian' | 'heal';
|
|
124
|
+
readonly operation: 'ingest' | 'librarian' | 'heal' | 'prune';
|
|
102
125
|
readonly entityId: string;
|
|
103
|
-
constructor(operation: 'ingest' | 'librarian' | 'heal', entityId: string);
|
|
126
|
+
constructor(operation: 'ingest' | 'librarian' | 'heal' | 'prune', entityId: string);
|
|
104
127
|
}
|
|
105
128
|
|
|
106
129
|
declare class WikiMemory {
|
|
@@ -154,4 +177,4 @@ declare class WikiMemory {
|
|
|
154
177
|
}>;
|
|
155
178
|
}
|
|
156
179
|
|
|
157
|
-
export { type EntityStatus as E, type FormattedMemoryDump as F, type LLMProvider as L, type MemoryDump as M, type WikiOptions as W, WikiMemory as a, type ExtractedFact as b, type ExtractedTask as c, type
|
|
180
|
+
export { type EntityStatus as E, type FormattedMemoryDump as F, type LLMProvider as L, type MemoryDump as M, type WikiOptions as W, WikiMemory as a, type ExtractedFact as b, type ExtractedTask as c, type FormatContextOptions as d, type MemoryBundle as e, WikiBusyError as f, type WikiCheckpoint as g, type WikiConfig as h, type WikiEvent as i, type WikiFact as j, type WikiTask as k };
|
package/dist/index.d.mts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as SQLite from 'expo-sqlite';
|
|
2
|
-
import { M as MemoryDump, F as FormattedMemoryDump, W as WikiOptions, a as WikiMemory } from './WikiMemory-
|
|
3
|
-
export { E as EntityStatus, b as ExtractedFact, c as ExtractedTask, L as LLMProvider,
|
|
2
|
+
import { M as MemoryDump, F as FormattedMemoryDump, W as WikiOptions, a as WikiMemory } from './WikiMemory-CjlQ68X0.mjs';
|
|
3
|
+
export { E as EntityStatus, b as ExtractedFact, c as ExtractedTask, d as FormatContextOptions, L as LLMProvider, e as MemoryBundle, f as WikiBusyError, g as WikiCheckpoint, h as WikiConfig, i as WikiEvent, j as WikiFact, k as WikiTask } from './WikiMemory-CjlQ68X0.mjs';
|
|
4
4
|
|
|
5
5
|
declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
|
|
6
6
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as SQLite from 'expo-sqlite';
|
|
2
|
-
import { M as MemoryDump, F as FormattedMemoryDump, W as WikiOptions, a as WikiMemory } from './WikiMemory-
|
|
3
|
-
export { E as EntityStatus, b as ExtractedFact, c as ExtractedTask, L as LLMProvider,
|
|
2
|
+
import { M as MemoryDump, F as FormattedMemoryDump, W as WikiOptions, a as WikiMemory } from './WikiMemory-CjlQ68X0.js';
|
|
3
|
+
export { E as EntityStatus, b as ExtractedFact, c as ExtractedTask, d as FormatContextOptions, L as LLMProvider, e as MemoryBundle, f as WikiBusyError, g as WikiCheckpoint, h as WikiConfig, i as WikiEvent, j as WikiFact, k as WikiTask } from './WikiMemory-CjlQ68X0.js';
|
|
4
4
|
|
|
5
5
|
declare function formatMemoryDump(dump: MemoryDump): FormattedMemoryDump;
|
|
6
6
|
|
package/dist/index.js
CHANGED
|
@@ -58,7 +58,8 @@ async function setupDatabase(db, prefix) {
|
|
|
58
58
|
body,
|
|
59
59
|
tags,
|
|
60
60
|
content='${prefix}entries',
|
|
61
|
-
content_rowid='rowid'
|
|
61
|
+
content_rowid='rowid',
|
|
62
|
+
tokenize='porter unicode61'
|
|
62
63
|
);
|
|
63
64
|
|
|
64
65
|
-- Triggers to keep FTS5 in sync with entries
|
|
@@ -373,6 +374,45 @@ var WikiMemory = class {
|
|
|
373
374
|
}
|
|
374
375
|
async setup() {
|
|
375
376
|
await setupDatabase(this.db, this.prefix);
|
|
377
|
+
const ftsMeta = await this.db.getFirstAsync(
|
|
378
|
+
`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
|
|
379
|
+
[`${this.prefix}entries_fts`]
|
|
380
|
+
);
|
|
381
|
+
const hasPorterTokenizer = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
|
|
382
|
+
if (ftsMeta?.sql && !hasPorterTokenizer) {
|
|
383
|
+
await this.db.withTransactionAsync(async () => {
|
|
384
|
+
await this.db.execAsync(`
|
|
385
|
+
DROP TRIGGER IF EXISTS ${this.prefix}entries_ai;
|
|
386
|
+
DROP TRIGGER IF EXISTS ${this.prefix}entries_ad;
|
|
387
|
+
DROP TRIGGER IF EXISTS ${this.prefix}entries_au;
|
|
388
|
+
DROP TABLE IF EXISTS ${this.prefix}entries_fts;
|
|
389
|
+
CREATE VIRTUAL TABLE ${this.prefix}entries_fts USING fts5(
|
|
390
|
+
title,
|
|
391
|
+
body,
|
|
392
|
+
tags,
|
|
393
|
+
content='${this.prefix}entries',
|
|
394
|
+
content_rowid='rowid',
|
|
395
|
+
tokenize='porter unicode61'
|
|
396
|
+
);
|
|
397
|
+
INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
|
|
398
|
+
SELECT rowid, title, body, tags FROM ${this.prefix}entries;
|
|
399
|
+
CREATE TRIGGER ${this.prefix}entries_ai AFTER INSERT ON ${this.prefix}entries BEGIN
|
|
400
|
+
INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
|
|
401
|
+
VALUES (new.rowid, new.title, new.body, new.tags);
|
|
402
|
+
END;
|
|
403
|
+
CREATE TRIGGER ${this.prefix}entries_ad AFTER DELETE ON ${this.prefix}entries BEGIN
|
|
404
|
+
INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
|
|
405
|
+
VALUES ('delete', old.rowid, old.title, old.body, old.tags);
|
|
406
|
+
END;
|
|
407
|
+
CREATE TRIGGER ${this.prefix}entries_au AFTER UPDATE ON ${this.prefix}entries BEGIN
|
|
408
|
+
INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
|
|
409
|
+
VALUES ('delete', old.rowid, old.title, old.body, old.tags);
|
|
410
|
+
INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
|
|
411
|
+
VALUES (new.rowid, new.title, new.body, new.tags);
|
|
412
|
+
END;
|
|
413
|
+
`);
|
|
414
|
+
});
|
|
415
|
+
}
|
|
376
416
|
const rows = await this.db.getAllAsync(`
|
|
377
417
|
SELECT rowid, source_ref FROM ${this.prefix}entries
|
|
378
418
|
WHERE source_ref IS NOT NULL
|
|
@@ -397,9 +437,35 @@ var WikiMemory = class {
|
|
|
397
437
|
});
|
|
398
438
|
}
|
|
399
439
|
formatSearchQuery(query) {
|
|
400
|
-
const
|
|
401
|
-
|
|
402
|
-
|
|
440
|
+
const normalizeTokens = (value) => value.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3);
|
|
441
|
+
const baseTokens = normalizeTokens(query);
|
|
442
|
+
if (baseTokens.length === 0) return "";
|
|
443
|
+
const synonymMap = this.options.config?.synonymMap;
|
|
444
|
+
const expanded = [];
|
|
445
|
+
const seen = /* @__PURE__ */ new Set();
|
|
446
|
+
const pushNormalized = (value) => {
|
|
447
|
+
for (const token of normalizeTokens(value)) {
|
|
448
|
+
if (expanded.length >= 12) return false;
|
|
449
|
+
if (seen.has(token)) continue;
|
|
450
|
+
seen.add(token);
|
|
451
|
+
expanded.push(token);
|
|
452
|
+
}
|
|
453
|
+
return true;
|
|
454
|
+
};
|
|
455
|
+
for (const t of baseTokens) {
|
|
456
|
+
if (!pushNormalized(t)) break;
|
|
457
|
+
if (synonymMap) {
|
|
458
|
+
const synonyms = synonymMap[t];
|
|
459
|
+
if (Array.isArray(synonyms)) {
|
|
460
|
+
for (const s of synonyms) {
|
|
461
|
+
if (typeof s === "string") {
|
|
462
|
+
if (!pushNormalized(s)) break;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
return expanded.map((t) => `"${t}"*`).join(" OR ");
|
|
403
469
|
}
|
|
404
470
|
async read(entityId, query) {
|
|
405
471
|
const ftsQuery = this.formatSearchQuery(query);
|
|
@@ -765,7 +831,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
765
831
|
if (factIdChunk.length === 0) continue;
|
|
766
832
|
const placeholders = factIdChunk.map(() => "?").join(", ");
|
|
767
833
|
const existingFacts = await this.db.getAllAsync(
|
|
768
|
-
`SELECT id, entity_id FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
834
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
769
835
|
factIdChunk
|
|
770
836
|
);
|
|
771
837
|
for (const existingFact of existingFacts) {
|
|
@@ -774,22 +840,27 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
774
840
|
}
|
|
775
841
|
for (const fact of bundle.facts) {
|
|
776
842
|
const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
|
|
843
|
+
const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
|
|
777
844
|
const existing = existingFactsById.get(fact.id);
|
|
778
845
|
if (existing) {
|
|
779
846
|
if (existing.entity_id !== entityId) {
|
|
780
847
|
this._warnCrossEntityCollision("entry", fact.id, existing.entity_id, entityId);
|
|
781
848
|
continue;
|
|
782
849
|
}
|
|
783
|
-
if (merge)
|
|
850
|
+
if (merge) {
|
|
851
|
+
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
852
|
+
}
|
|
784
853
|
await this.db.runAsync(
|
|
785
854
|
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ? WHERE id = ?`,
|
|
786
|
-
[entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at,
|
|
855
|
+
[entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
|
|
787
856
|
);
|
|
857
|
+
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
788
858
|
} else {
|
|
789
859
|
await this.db.runAsync(
|
|
790
860
|
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
791
|
-
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at,
|
|
861
|
+
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
|
|
792
862
|
);
|
|
863
|
+
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
793
864
|
}
|
|
794
865
|
}
|
|
795
866
|
const taskIds = bundle.tasks.map((task) => task.id);
|
|
@@ -800,7 +871,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
800
871
|
if (taskIdChunk.length === 0) continue;
|
|
801
872
|
const placeholders = taskIdChunk.map(() => "?").join(", ");
|
|
802
873
|
const existingTasks = await this.db.getAllAsync(
|
|
803
|
-
`SELECT id, entity_id FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
874
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
804
875
|
taskIdChunk
|
|
805
876
|
);
|
|
806
877
|
for (const existingTask of existingTasks) {
|
|
@@ -808,22 +879,27 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
808
879
|
}
|
|
809
880
|
}
|
|
810
881
|
for (const task of bundle.tasks) {
|
|
882
|
+
const safeUpdatedAt = Number.isFinite(task.updated_at) ? task.updated_at : 0;
|
|
811
883
|
const existing = existingTasksById.get(task.id);
|
|
812
884
|
if (existing) {
|
|
813
885
|
if (existing.entity_id !== entityId) {
|
|
814
886
|
this._warnCrossEntityCollision("task", task.id, existing.entity_id, entityId);
|
|
815
887
|
continue;
|
|
816
888
|
}
|
|
817
|
-
if (merge)
|
|
889
|
+
if (merge) {
|
|
890
|
+
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
891
|
+
}
|
|
818
892
|
await this.db.runAsync(
|
|
819
893
|
`UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
|
|
820
|
-
[entityId, task.description, task.status, task.priority, task.created_at,
|
|
894
|
+
[entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at, task.id]
|
|
821
895
|
);
|
|
896
|
+
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
822
897
|
} else {
|
|
823
898
|
await this.db.runAsync(
|
|
824
899
|
`INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
825
|
-
[task.id, entityId, task.description, task.status, task.priority, task.created_at,
|
|
900
|
+
[task.id, entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at]
|
|
826
901
|
);
|
|
902
|
+
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
827
903
|
}
|
|
828
904
|
}
|
|
829
905
|
for (const event of bundle.events) {
|
package/dist/index.mjs
CHANGED
|
@@ -29,7 +29,8 @@ async function setupDatabase(db, prefix) {
|
|
|
29
29
|
body,
|
|
30
30
|
tags,
|
|
31
31
|
content='${prefix}entries',
|
|
32
|
-
content_rowid='rowid'
|
|
32
|
+
content_rowid='rowid',
|
|
33
|
+
tokenize='porter unicode61'
|
|
33
34
|
);
|
|
34
35
|
|
|
35
36
|
-- Triggers to keep FTS5 in sync with entries
|
|
@@ -344,6 +345,45 @@ var WikiMemory = class {
|
|
|
344
345
|
}
|
|
345
346
|
async setup() {
|
|
346
347
|
await setupDatabase(this.db, this.prefix);
|
|
348
|
+
const ftsMeta = await this.db.getFirstAsync(
|
|
349
|
+
`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
|
|
350
|
+
[`${this.prefix}entries_fts`]
|
|
351
|
+
);
|
|
352
|
+
const hasPorterTokenizer = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
|
|
353
|
+
if (ftsMeta?.sql && !hasPorterTokenizer) {
|
|
354
|
+
await this.db.withTransactionAsync(async () => {
|
|
355
|
+
await this.db.execAsync(`
|
|
356
|
+
DROP TRIGGER IF EXISTS ${this.prefix}entries_ai;
|
|
357
|
+
DROP TRIGGER IF EXISTS ${this.prefix}entries_ad;
|
|
358
|
+
DROP TRIGGER IF EXISTS ${this.prefix}entries_au;
|
|
359
|
+
DROP TABLE IF EXISTS ${this.prefix}entries_fts;
|
|
360
|
+
CREATE VIRTUAL TABLE ${this.prefix}entries_fts USING fts5(
|
|
361
|
+
title,
|
|
362
|
+
body,
|
|
363
|
+
tags,
|
|
364
|
+
content='${this.prefix}entries',
|
|
365
|
+
content_rowid='rowid',
|
|
366
|
+
tokenize='porter unicode61'
|
|
367
|
+
);
|
|
368
|
+
INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
|
|
369
|
+
SELECT rowid, title, body, tags FROM ${this.prefix}entries;
|
|
370
|
+
CREATE TRIGGER ${this.prefix}entries_ai AFTER INSERT ON ${this.prefix}entries BEGIN
|
|
371
|
+
INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
|
|
372
|
+
VALUES (new.rowid, new.title, new.body, new.tags);
|
|
373
|
+
END;
|
|
374
|
+
CREATE TRIGGER ${this.prefix}entries_ad AFTER DELETE ON ${this.prefix}entries BEGIN
|
|
375
|
+
INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
|
|
376
|
+
VALUES ('delete', old.rowid, old.title, old.body, old.tags);
|
|
377
|
+
END;
|
|
378
|
+
CREATE TRIGGER ${this.prefix}entries_au AFTER UPDATE ON ${this.prefix}entries BEGIN
|
|
379
|
+
INSERT INTO ${this.prefix}entries_fts(${this.prefix}entries_fts, rowid, title, body, tags)
|
|
380
|
+
VALUES ('delete', old.rowid, old.title, old.body, old.tags);
|
|
381
|
+
INSERT INTO ${this.prefix}entries_fts(rowid, title, body, tags)
|
|
382
|
+
VALUES (new.rowid, new.title, new.body, new.tags);
|
|
383
|
+
END;
|
|
384
|
+
`);
|
|
385
|
+
});
|
|
386
|
+
}
|
|
347
387
|
const rows = await this.db.getAllAsync(`
|
|
348
388
|
SELECT rowid, source_ref FROM ${this.prefix}entries
|
|
349
389
|
WHERE source_ref IS NOT NULL
|
|
@@ -368,9 +408,35 @@ var WikiMemory = class {
|
|
|
368
408
|
});
|
|
369
409
|
}
|
|
370
410
|
formatSearchQuery(query) {
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
|
|
411
|
+
const normalizeTokens = (value) => value.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3);
|
|
412
|
+
const baseTokens = normalizeTokens(query);
|
|
413
|
+
if (baseTokens.length === 0) return "";
|
|
414
|
+
const synonymMap = this.options.config?.synonymMap;
|
|
415
|
+
const expanded = [];
|
|
416
|
+
const seen = /* @__PURE__ */ new Set();
|
|
417
|
+
const pushNormalized = (value) => {
|
|
418
|
+
for (const token of normalizeTokens(value)) {
|
|
419
|
+
if (expanded.length >= 12) return false;
|
|
420
|
+
if (seen.has(token)) continue;
|
|
421
|
+
seen.add(token);
|
|
422
|
+
expanded.push(token);
|
|
423
|
+
}
|
|
424
|
+
return true;
|
|
425
|
+
};
|
|
426
|
+
for (const t of baseTokens) {
|
|
427
|
+
if (!pushNormalized(t)) break;
|
|
428
|
+
if (synonymMap) {
|
|
429
|
+
const synonyms = synonymMap[t];
|
|
430
|
+
if (Array.isArray(synonyms)) {
|
|
431
|
+
for (const s of synonyms) {
|
|
432
|
+
if (typeof s === "string") {
|
|
433
|
+
if (!pushNormalized(s)) break;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
return expanded.map((t) => `"${t}"*`).join(" OR ");
|
|
374
440
|
}
|
|
375
441
|
async read(entityId, query) {
|
|
376
442
|
const ftsQuery = this.formatSearchQuery(query);
|
|
@@ -736,7 +802,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
736
802
|
if (factIdChunk.length === 0) continue;
|
|
737
803
|
const placeholders = factIdChunk.map(() => "?").join(", ");
|
|
738
804
|
const existingFacts = await this.db.getAllAsync(
|
|
739
|
-
`SELECT id, entity_id FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
805
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
740
806
|
factIdChunk
|
|
741
807
|
);
|
|
742
808
|
for (const existingFact of existingFacts) {
|
|
@@ -745,22 +811,27 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
745
811
|
}
|
|
746
812
|
for (const fact of bundle.facts) {
|
|
747
813
|
const tagsJson = JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
|
|
814
|
+
const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
|
|
748
815
|
const existing = existingFactsById.get(fact.id);
|
|
749
816
|
if (existing) {
|
|
750
817
|
if (existing.entity_id !== entityId) {
|
|
751
818
|
this._warnCrossEntityCollision("entry", fact.id, existing.entity_id, entityId);
|
|
752
819
|
continue;
|
|
753
820
|
}
|
|
754
|
-
if (merge)
|
|
821
|
+
if (merge) {
|
|
822
|
+
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
823
|
+
}
|
|
755
824
|
await this.db.runAsync(
|
|
756
825
|
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ? WHERE id = ?`,
|
|
757
|
-
[entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at,
|
|
826
|
+
[entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
|
|
758
827
|
);
|
|
828
|
+
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
759
829
|
} else {
|
|
760
830
|
await this.db.runAsync(
|
|
761
831
|
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
762
|
-
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at,
|
|
832
|
+
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, fact.source_type, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
|
|
763
833
|
);
|
|
834
|
+
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
764
835
|
}
|
|
765
836
|
}
|
|
766
837
|
const taskIds = bundle.tasks.map((task) => task.id);
|
|
@@ -771,7 +842,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
771
842
|
if (taskIdChunk.length === 0) continue;
|
|
772
843
|
const placeholders = taskIdChunk.map(() => "?").join(", ");
|
|
773
844
|
const existingTasks = await this.db.getAllAsync(
|
|
774
|
-
`SELECT id, entity_id FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
845
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
775
846
|
taskIdChunk
|
|
776
847
|
);
|
|
777
848
|
for (const existingTask of existingTasks) {
|
|
@@ -779,22 +850,27 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
779
850
|
}
|
|
780
851
|
}
|
|
781
852
|
for (const task of bundle.tasks) {
|
|
853
|
+
const safeUpdatedAt = Number.isFinite(task.updated_at) ? task.updated_at : 0;
|
|
782
854
|
const existing = existingTasksById.get(task.id);
|
|
783
855
|
if (existing) {
|
|
784
856
|
if (existing.entity_id !== entityId) {
|
|
785
857
|
this._warnCrossEntityCollision("task", task.id, existing.entity_id, entityId);
|
|
786
858
|
continue;
|
|
787
859
|
}
|
|
788
|
-
if (merge)
|
|
860
|
+
if (merge) {
|
|
861
|
+
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
862
|
+
}
|
|
789
863
|
await this.db.runAsync(
|
|
790
864
|
`UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
|
|
791
|
-
[entityId, task.description, task.status, task.priority, task.created_at,
|
|
865
|
+
[entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at, task.id]
|
|
792
866
|
);
|
|
867
|
+
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
793
868
|
} else {
|
|
794
869
|
await this.db.runAsync(
|
|
795
870
|
`INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
796
|
-
[task.id, entityId, task.description, task.status, task.priority, task.created_at,
|
|
871
|
+
[task.id, entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at]
|
|
797
872
|
);
|
|
873
|
+
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
798
874
|
}
|
|
799
875
|
}
|
|
800
876
|
for (const event of bundle.events) {
|
package/dist/react/index.d.mts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import { ReactNode } from 'react';
|
|
3
|
-
import { a as WikiMemory,
|
|
3
|
+
import { a as WikiMemory, e as MemoryBundle, i as WikiEvent, M as MemoryDump } from '../WikiMemory-CjlQ68X0.mjs';
|
|
4
4
|
import 'expo-sqlite';
|
|
5
5
|
|
|
6
6
|
declare function WikiProvider({ wiki, children }: {
|
package/dist/react/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as react_jsx_runtime from 'react/jsx-runtime';
|
|
2
2
|
import { ReactNode } from 'react';
|
|
3
|
-
import { a as WikiMemory,
|
|
3
|
+
import { a as WikiMemory, e as MemoryBundle, i as WikiEvent, M as MemoryDump } from '../WikiMemory-CjlQ68X0.js';
|
|
4
4
|
import 'expo-sqlite';
|
|
5
5
|
|
|
6
6
|
declare function WikiProvider({ wiki, children }: {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@equationalapplications/expo-llm-wiki",
|
|
3
|
-
"version": "2.0
|
|
3
|
+
"version": "2.2.0",
|
|
4
4
|
"description": "A standalone, general-purpose React Native/Expo package for LLM Wiki Memory.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
"scripts": {
|
|
21
21
|
"build": "tsup src/index.ts src/react/index.ts --format cjs,esm --dts --external react",
|
|
22
22
|
"dev": "tsup src/index.ts src/react/index.ts --format cjs,esm --dts --external react --watch",
|
|
23
|
+
"typecheck": "tsc --noEmit",
|
|
23
24
|
"test": "vitest run",
|
|
24
25
|
"test:watch": "vitest"
|
|
25
26
|
},
|
|
@@ -51,12 +52,17 @@
|
|
|
51
52
|
"optional": true
|
|
52
53
|
}
|
|
53
54
|
},
|
|
55
|
+
"engines": {
|
|
56
|
+
"node": ">=20"
|
|
57
|
+
},
|
|
54
58
|
"devDependencies": {
|
|
55
59
|
"@semantic-release/changelog": "^6.0.0",
|
|
56
60
|
"@semantic-release/git": "^10.0.0",
|
|
61
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
57
62
|
"@types/node": "^20.0.0",
|
|
58
63
|
"@types/react": "^19.0.0",
|
|
59
64
|
"@vitest/coverage-v8": "^4.1.5",
|
|
65
|
+
"better-sqlite3": "^12.9.0",
|
|
60
66
|
"expo-sqlite": "^55.0.15",
|
|
61
67
|
"react": "19.2.0",
|
|
62
68
|
"semantic-release": "^24.0.0",
|