@steno-ai/engine 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/storage.d.ts +29 -2
- package/dist/adapters/storage.d.ts.map +1 -1
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +23 -0
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/types.d.ts +2 -0
- package/dist/extraction/types.d.ts.map +1 -1
- package/dist/identity/index.d.ts +2 -0
- package/dist/identity/index.d.ts.map +1 -0
- package/dist/identity/index.js +2 -0
- package/dist/identity/index.js.map +1 -0
- package/dist/identity/resolver.d.ts +31 -0
- package/dist/identity/resolver.d.ts.map +1 -0
- package/dist/identity/resolver.js +122 -0
- package/dist/identity/resolver.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/models/entity.d.ts +32 -0
- package/dist/models/entity.d.ts.map +1 -1
- package/dist/models/entity.js +11 -0
- package/dist/models/entity.js.map +1 -1
- package/dist/retrieval/graph-traversal.d.ts +4 -1
- package/dist/retrieval/graph-traversal.d.ts.map +1 -1
- package/dist/retrieval/graph-traversal.js +6 -3
- package/dist/retrieval/graph-traversal.js.map +1 -1
- package/dist/retrieval/search.d.ts.map +1 -1
- package/dist/retrieval/search.js +56 -3
- package/dist/retrieval/search.js.map +1 -1
- package/dist/retrieval/types.d.ts +1 -0
- package/dist/retrieval/types.d.ts.map +1 -1
- package/dist/retrieval/types.js.map +1 -1
- package/package.json +1 -1
- package/src/adapters/storage.ts +35 -2
- package/src/extraction/pipeline.ts +34 -0
- package/src/extraction/types.ts +2 -0
- package/src/identity/index.ts +1 -0
- package/src/identity/resolver.ts +149 -0
- package/src/index.ts +1 -0
- package/src/models/entity.ts +13 -0
- package/src/retrieval/graph-traversal.ts +7 -4
- package/src/retrieval/search.ts +58 -3
- package/src/retrieval/types.ts +1 -0
- package/src/adapters/cache.d.ts +0 -9
- package/src/adapters/cache.d.ts.map +0 -1
- package/src/adapters/cache.js +0 -2
- package/src/adapters/cache.js.map +0 -1
- package/src/adapters/embedding.d.ts +0 -7
- package/src/adapters/embedding.d.ts.map +0 -1
- package/src/adapters/embedding.js +0 -2
- package/src/adapters/embedding.js.map +0 -1
- package/src/adapters/llm.d.ts +0 -19
- package/src/adapters/llm.d.ts.map +0 -1
- package/src/adapters/llm.js +0 -2
- package/src/adapters/llm.js.map +0 -1
- package/src/adapters/perplexity-embedding.d.ts +0 -24
- package/src/adapters/perplexity-embedding.d.ts.map +0 -1
- package/src/adapters/perplexity-embedding.js +0 -78
- package/src/adapters/perplexity-embedding.js.map +0 -1
- package/src/adapters/storage.d.ts +0 -173
- package/src/adapters/storage.d.ts.map +0 -1
- package/src/adapters/storage.js +0 -2
- package/src/adapters/storage.js.map +0 -1
- package/src/config.d.ts +0 -296
- package/src/config.d.ts.map +0 -1
- package/src/config.js +0 -92
- package/src/config.js.map +0 -1
- package/src/extraction/contradiction.d.ts +0 -15
- package/src/extraction/contradiction.d.ts.map +0 -1
- package/src/extraction/contradiction.js +0 -23
- package/src/extraction/contradiction.js.map +0 -1
- package/src/extraction/cross-linker.d.ts +0 -23
- package/src/extraction/cross-linker.d.ts.map +0 -1
- package/src/extraction/cross-linker.js +0 -146
- package/src/extraction/cross-linker.js.map +0 -1
- package/src/extraction/dedup.d.ts +0 -12
- package/src/extraction/dedup.d.ts.map +0 -1
- package/src/extraction/dedup.js +0 -93
- package/src/extraction/dedup.js.map +0 -1
- package/src/extraction/entity-extractor.d.ts +0 -30
- package/src/extraction/entity-extractor.d.ts.map +0 -1
- package/src/extraction/entity-extractor.js +0 -145
- package/src/extraction/entity-extractor.js.map +0 -1
- package/src/extraction/hasher.d.ts +0 -5
- package/src/extraction/hasher.d.ts.map +0 -1
- package/src/extraction/hasher.js +0 -8
- package/src/extraction/hasher.js.map +0 -1
- package/src/extraction/heuristic.d.ts +0 -3
- package/src/extraction/heuristic.d.ts.map +0 -1
- package/src/extraction/heuristic.js +0 -282
- package/src/extraction/heuristic.js.map +0 -1
- package/src/extraction/llm-extractor.d.ts +0 -23
- package/src/extraction/llm-extractor.d.ts.map +0 -1
- package/src/extraction/llm-extractor.js +0 -240
- package/src/extraction/llm-extractor.js.map +0 -1
- package/src/extraction/pipeline.d.ts +0 -30
- package/src/extraction/pipeline.d.ts.map +0 -1
- package/src/extraction/pipeline.js +0 -413
- package/src/extraction/pipeline.js.map +0 -1
- package/src/extraction/prompts.d.ts +0 -28
- package/src/extraction/prompts.d.ts.map +0 -1
- package/src/extraction/prompts.js +0 -205
- package/src/extraction/prompts.js.map +0 -1
- package/src/extraction/sliding-window.d.ts +0 -41
- package/src/extraction/sliding-window.d.ts.map +0 -1
- package/src/extraction/sliding-window.js +0 -84
- package/src/extraction/sliding-window.js.map +0 -1
- package/src/extraction/types.d.ts +0 -80
- package/src/extraction/types.d.ts.map +0 -1
- package/src/extraction/types.js +0 -2
- package/src/extraction/types.js.map +0 -1
- package/src/feedback/tracker.d.ts +0 -25
- package/src/feedback/tracker.d.ts.map +0 -1
- package/src/feedback/tracker.js +0 -90
- package/src/feedback/tracker.js.map +0 -1
- package/src/models/api-key.d.ts +0 -54
- package/src/models/api-key.d.ts.map +0 -1
- package/src/models/api-key.js +0 -21
- package/src/models/api-key.js.map +0 -1
- package/src/models/edge.d.ts +0 -78
- package/src/models/edge.d.ts.map +0 -1
- package/src/models/edge.js +0 -29
- package/src/models/edge.js.map +0 -1
- package/src/models/entity.d.ts +0 -60
- package/src/models/entity.d.ts.map +0 -1
- package/src/models/entity.js +0 -22
- package/src/models/entity.js.map +0 -1
- package/src/models/extraction.d.ts +0 -111
- package/src/models/extraction.d.ts.map +0 -1
- package/src/models/extraction.js +0 -40
- package/src/models/extraction.js.map +0 -1
- package/src/models/fact-entity.d.ts +0 -33
- package/src/models/fact-entity.d.ts.map +0 -1
- package/src/models/fact-entity.js +0 -14
- package/src/models/fact-entity.js.map +0 -1
- package/src/models/fact.d.ts +0 -191
- package/src/models/fact.d.ts.map +0 -1
- package/src/models/fact.js +0 -72
- package/src/models/fact.js.map +0 -1
- package/src/models/index.d.ts +0 -13
- package/src/models/index.d.ts.map +0 -1
- package/src/models/index.js +0 -13
- package/src/models/index.js.map +0 -1
- package/src/models/memory-access.d.ts +0 -89
- package/src/models/memory-access.d.ts.map +0 -1
- package/src/models/memory-access.js +0 -33
- package/src/models/memory-access.js.map +0 -1
- package/src/models/session.d.ts +0 -60
- package/src/models/session.d.ts.map +0 -1
- package/src/models/session.js +0 -23
- package/src/models/session.js.map +0 -1
- package/src/models/tenant.d.ts +0 -448
- package/src/models/tenant.d.ts.map +0 -1
- package/src/models/tenant.js +0 -23
- package/src/models/tenant.js.map +0 -1
- package/src/models/trigger.d.ts +0 -87
- package/src/models/trigger.d.ts.map +0 -1
- package/src/models/trigger.js +0 -41
- package/src/models/trigger.js.map +0 -1
- package/src/models/usage-record.d.ts +0 -37
- package/src/models/usage-record.d.ts.map +0 -1
- package/src/models/usage-record.js +0 -14
- package/src/models/usage-record.js.map +0 -1
- package/src/models/webhook.d.ts +0 -50
- package/src/models/webhook.d.ts.map +0 -1
- package/src/models/webhook.js +0 -25
- package/src/models/webhook.js.map +0 -1
- package/src/retrieval/compound-search.d.ts +0 -13
- package/src/retrieval/compound-search.d.ts.map +0 -1
- package/src/retrieval/compound-search.js +0 -87
- package/src/retrieval/compound-search.js.map +0 -1
- package/src/retrieval/contradiction-surfacer.d.ts +0 -18
- package/src/retrieval/contradiction-surfacer.d.ts.map +0 -1
- package/src/retrieval/contradiction-surfacer.js +0 -64
- package/src/retrieval/contradiction-surfacer.js.map +0 -1
- package/src/retrieval/embedding-cache.d.ts +0 -17
- package/src/retrieval/embedding-cache.d.ts.map +0 -1
- package/src/retrieval/embedding-cache.js +0 -56
- package/src/retrieval/embedding-cache.js.map +0 -1
- package/src/retrieval/fusion.d.ts +0 -27
- package/src/retrieval/fusion.d.ts.map +0 -1
- package/src/retrieval/fusion.js +0 -87
- package/src/retrieval/fusion.js.map +0 -1
- package/src/retrieval/graph-traversal.d.ts +0 -29
- package/src/retrieval/graph-traversal.d.ts.map +0 -1
- package/src/retrieval/graph-traversal.js +0 -208
- package/src/retrieval/graph-traversal.js.map +0 -1
- package/src/retrieval/query-expansion.d.ts +0 -20
- package/src/retrieval/query-expansion.d.ts.map +0 -1
- package/src/retrieval/query-expansion.js +0 -76
- package/src/retrieval/query-expansion.js.map +0 -1
- package/src/retrieval/reranker.d.ts +0 -15
- package/src/retrieval/reranker.d.ts.map +0 -1
- package/src/retrieval/reranker.js +0 -47
- package/src/retrieval/reranker.js.map +0 -1
- package/src/retrieval/salience-scorer.d.ts +0 -15
- package/src/retrieval/salience-scorer.d.ts.map +0 -1
- package/src/retrieval/salience-scorer.js +0 -41
- package/src/retrieval/salience-scorer.js.map +0 -1
- package/src/retrieval/search.d.ts +0 -21
- package/src/retrieval/search.d.ts.map +0 -1
- package/src/retrieval/search.js +0 -228
- package/src/retrieval/search.js.map +0 -1
- package/src/retrieval/temporal-scorer.d.ts +0 -18
- package/src/retrieval/temporal-scorer.d.ts.map +0 -1
- package/src/retrieval/temporal-scorer.js +0 -106
- package/src/retrieval/temporal-scorer.js.map +0 -1
- package/src/retrieval/trigger-matcher.d.ts +0 -18
- package/src/retrieval/trigger-matcher.d.ts.map +0 -1
- package/src/retrieval/trigger-matcher.js +0 -134
- package/src/retrieval/trigger-matcher.js.map +0 -1
- package/src/retrieval/types.d.ts +0 -70
- package/src/retrieval/types.d.ts.map +0 -1
- package/src/retrieval/types.js +0 -9
- package/src/retrieval/types.js.map +0 -1
- package/src/retrieval/vector-search.d.ts +0 -5
- package/src/retrieval/vector-search.d.ts.map +0 -1
- package/src/retrieval/vector-search.js +0 -24
- package/src/retrieval/vector-search.js.map +0 -1
- package/src/salience/decay.d.ts +0 -9
- package/src/salience/decay.d.ts.map +0 -1
- package/src/salience/decay.js +0 -15
- package/src/salience/decay.js.map +0 -1
- package/src/scratchpad/scratchpad.d.ts +0 -23
- package/src/scratchpad/scratchpad.d.ts.map +0 -1
- package/src/scratchpad/scratchpad.js +0 -137
- package/src/scratchpad/scratchpad.js.map +0 -1
- package/src/sessions/manager.d.ts +0 -11
- package/src/sessions/manager.d.ts.map +0 -1
- package/src/sessions/manager.js +0 -63
- package/src/sessions/manager.js.map +0 -1
package/src/config.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,SAAS,aAA2B,CAAC;AAMlD,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;EAK5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAIjC,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAE7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAG9D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiB5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,MAAM,+CAAgD,CAAC;AACpE,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;AAG5C,eAAO,MAAM,cAAc,oCAAqC,CAAC;AACjE,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D,eAAO,MAAM,UAAU,6CAA8C,CAAC;AACtE,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEpD,eAAO,MAAM,sBAAsB,uDAAwD,CAAC;AAC5F,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE1E,eAAO,MAAM,YAAY,+EAOf,CAAC;AACX,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,gBAAgB,kDAAmD,CAAC;AACjF,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE/D,eAAO,MAAM,qBAAqB,gEAA+C,CAAC;AAClF,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC;AAExE,eAAO,MAAM,UAAU,yDAA0D,CAAC;AAClF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,eAAO,MAAM,UAAU,kHASb,CAAC;AACX,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,eAAO,MAAM,mBAAmB,qEAMtB,CAAC;AACX,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEpE,eAAO,MAAM,WAAW,oFAQd,CAAC;AACX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD,eAAO,MAAM,KAAK,iDAAkD,CAAC;AACrE,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;AAE1C,eAAO,MAAM,cAAc,qCAAsC,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE1D,eAAO,MAAM,cAAc,6GAMjB,CAAC;AACX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D,eAAO,MAAM,YAAY,6CAA8C,CAAC;AACxE,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC"}
|
package/src/config.js
DELETED
|
@@ -1,92 +0,0 @@
|
|
|
1
|
-
import { z } from 'zod';
|
|
2
|
-
// Shared Zod helpers
|
|
3
|
-
export const unitFloat = z.number().min(0).max(1);
|
|
4
|
-
// ---------------------------------------------------------------------------
|
|
5
|
-
// Domain-scoped graph schemas — custom entity types with typed attributes
|
|
6
|
-
// ---------------------------------------------------------------------------
|
|
7
|
-
export const EntityFieldSchema = z.object({
|
|
8
|
-
name: z.string(),
|
|
9
|
-
type: z.enum(['string', 'number', 'boolean', 'date']),
|
|
10
|
-
description: z.string(),
|
|
11
|
-
required: z.boolean().default(false),
|
|
12
|
-
});
|
|
13
|
-
export const DomainEntityTypeSchema = z.object({
|
|
14
|
-
name: z.string(),
|
|
15
|
-
description: z.string(),
|
|
16
|
-
fields: z.array(EntityFieldSchema).default([]),
|
|
17
|
-
});
|
|
18
|
-
export const DomainSchemaSchema = z.object({
|
|
19
|
-
entityTypes: z.array(DomainEntityTypeSchema).default([]),
|
|
20
|
-
});
|
|
21
|
-
// Steno engine configuration schema
|
|
22
|
-
export const StenoConfigSchema = z.object({
|
|
23
|
-
embeddingModel: z.string().default('text-embedding-3-small'),
|
|
24
|
-
embeddingDim: z.number().int().positive().default(1536),
|
|
25
|
-
decayHalfLifeDays: z.number().positive().default(30),
|
|
26
|
-
decayNormalizationK: z.number().positive().default(50),
|
|
27
|
-
maxFactsPerScope: z.number().int().positive().default(10000),
|
|
28
|
-
retrievalWeights: z
|
|
29
|
-
.object({
|
|
30
|
-
vector: z.number().min(0).max(1).default(0.35),
|
|
31
|
-
keyword: z.number().min(0).max(1).default(0.15),
|
|
32
|
-
graph: z.number().min(0).max(1).default(0.2),
|
|
33
|
-
recency: z.number().min(0).max(1).default(0.15),
|
|
34
|
-
salience: z.number().min(0).max(1).default(0.15),
|
|
35
|
-
temporal: z.number().min(0).max(1).default(0.20),
|
|
36
|
-
})
|
|
37
|
-
.default({}),
|
|
38
|
-
domainSchema: DomainSchemaSchema.optional(),
|
|
39
|
-
});
|
|
40
|
-
export const SCOPES = ['user', 'agent', 'session', 'hive'];
|
|
41
|
-
// SESSION_SCOPES is a subset of SCOPES — sessions themselves cannot be scoped to another session
|
|
42
|
-
export const SESSION_SCOPES = ['user', 'agent', 'hive'];
|
|
43
|
-
export const OPERATIONS = ['create', 'update', 'invalidate'];
|
|
44
|
-
export const CONTRADICTION_STATUSES = ['none', 'active', 'resolved', 'superseded'];
|
|
45
|
-
export const SOURCE_TYPES = [
|
|
46
|
-
'conversation',
|
|
47
|
-
'document',
|
|
48
|
-
'url',
|
|
49
|
-
'raw_text',
|
|
50
|
-
'api',
|
|
51
|
-
'agent_self',
|
|
52
|
-
];
|
|
53
|
-
export const EXTRACTION_TIERS = ['heuristic', 'cheap_llm', 'smart_llm'];
|
|
54
|
-
export const EXTRACTION_TIERS_USED = [...EXTRACTION_TIERS, 'multi_tier'];
|
|
55
|
-
export const MODALITIES = ['text', 'image', 'audio', 'code', 'document'];
|
|
56
|
-
export const EDGE_TYPES = [
|
|
57
|
-
'associative',
|
|
58
|
-
'causal',
|
|
59
|
-
'temporal',
|
|
60
|
-
'contradictory',
|
|
61
|
-
'hierarchical',
|
|
62
|
-
'updates', // new fact supersedes old one (knowledge chain)
|
|
63
|
-
'extends', // new fact adds detail to old one
|
|
64
|
-
'derives', // new fact is inferred from combining others
|
|
65
|
-
];
|
|
66
|
-
export const EXTRACTION_STATUSES = [
|
|
67
|
-
'queued',
|
|
68
|
-
'processing',
|
|
69
|
-
'completed',
|
|
70
|
-
'failed',
|
|
71
|
-
'deduped',
|
|
72
|
-
];
|
|
73
|
-
export const INPUT_TYPES = [
|
|
74
|
-
'conversation',
|
|
75
|
-
'document',
|
|
76
|
-
'url',
|
|
77
|
-
'raw_text',
|
|
78
|
-
'image',
|
|
79
|
-
'audio',
|
|
80
|
-
'code',
|
|
81
|
-
];
|
|
82
|
-
export const PLANS = ['free', 'pro', 'scale', 'enterprise'];
|
|
83
|
-
export const API_KEY_SCOPES = ['read', 'write', 'admin'];
|
|
84
|
-
export const FEEDBACK_TYPES = [
|
|
85
|
-
'implicit_positive',
|
|
86
|
-
'implicit_negative',
|
|
87
|
-
'explicit_positive',
|
|
88
|
-
'explicit_negative',
|
|
89
|
-
'correction',
|
|
90
|
-
];
|
|
91
|
-
export const ENTITY_ROLES = ['subject', 'object', 'mentioned'];
|
|
92
|
-
//# sourceMappingURL=config.js.map
|
package/src/config.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,qBAAqB;AACrB,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAElD,8EAA8E;AAC9E,0EAA0E;AAC1E,8EAA8E;AAE9E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;IACrD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;CACrC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CAC/C,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CACzD,CAAC,CAAC;AAGH,oCAAoC;AACpC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC;IAC5D,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACvD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACpD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IAC5D,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QAC9C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QAC/C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;QAC5C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QAC/C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;QAChD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;KACjD,CAAC;SACD,OAAO,CAAC,EAAE,CAAC;IACd,YAAY,EAAE,kBAAkB,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAU,CAAC;AAGpE,iGAAiG;AACjG,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAU,CAAC;AAGjE,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAU,CAAC;AAGtE,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,CAAU,CAAC;AAG5F,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,cAAc;IACd,UAAU;IACV,KAAK;IACL,UAAU;IACV,KAAK;IACL,YAAY;CACJ,CAAC;AAGX,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,WAAW,CAAU,CAAC;AAGjF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,GAAG,gBAAgB,EAAE,YAAY,CAAU,CAAC;AAGlF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAU,CAAC;AAGlF,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,aAAa;IACb,QAAQ;IACR,UAAU;IACV,eAAe;IACf,cAAc;IACd,SAAS,EAAK,gDAAgD;IAC9D,SAAS,EAAK,kCAAkC;IAChD,SAAS,EAAK,6CAA6C;CACnD,CAAC;AAGX,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,QAAQ;IACR,YAAY;IACZ,WAAW;IACX,QAAQ;IACR,SAAS;CACD,CAAC;AAGX,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,cAAc;IACd,UAAU;IACV,KAAK;IACL,UAAU;IACV,OAAO;IACP,OAAO;IACP,MAAM;CACE,CAAC;AAGX,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,CAAU,CAAC;AAGrE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAU,CAAC;AAGlE,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,mBAAmB;IACnB,mBAAmB;IACnB,mBAAmB;IACnB,mBAAmB;IACnB,YAAY;CACJ,CAAC;AAGX,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAU,CAAC"}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import type { ExtractedFact } from './types.js';
|
|
2
|
-
import type { ContradictionStatus } from '../config.js';
|
|
3
|
-
export interface ContradictionResult {
|
|
4
|
-
fact: ExtractedFact;
|
|
5
|
-
contradictionStatus: ContradictionStatus;
|
|
6
|
-
contradictsId: string | null;
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* Process extracted facts and annotate contradictions.
|
|
10
|
-
* Facts with operation='contradict' get contradictionStatus='active'
|
|
11
|
-
* and contradictsId set to the fact they contradict.
|
|
12
|
-
* All other facts get contradictionStatus='none'.
|
|
13
|
-
*/
|
|
14
|
-
export declare function processContradictions(facts: ExtractedFact[]): ContradictionResult[];
|
|
15
|
-
//# sourceMappingURL=contradiction.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"contradiction.d.ts","sourceRoot":"","sources":["contradiction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAExD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,aAAa,CAAC;IACpB,mBAAmB,EAAE,mBAAmB,CAAC;IACzC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,aAAa,EAAE,GACrB,mBAAmB,EAAE,CAevB"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Process extracted facts and annotate contradictions.
|
|
3
|
-
* Facts with operation='contradict' get contradictionStatus='active'
|
|
4
|
-
* and contradictsId set to the fact they contradict.
|
|
5
|
-
* All other facts get contradictionStatus='none'.
|
|
6
|
-
*/
|
|
7
|
-
export function processContradictions(facts) {
|
|
8
|
-
return facts.map(fact => {
|
|
9
|
-
if (fact.operation === 'contradict' && fact.contradictsFactId) {
|
|
10
|
-
return {
|
|
11
|
-
fact,
|
|
12
|
-
contradictionStatus: 'active',
|
|
13
|
-
contradictsId: fact.contradictsFactId,
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
return {
|
|
17
|
-
fact,
|
|
18
|
-
contradictionStatus: 'none',
|
|
19
|
-
contradictsId: null,
|
|
20
|
-
};
|
|
21
|
-
});
|
|
22
|
-
}
|
|
23
|
-
//# sourceMappingURL=contradiction.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"contradiction.js","sourceRoot":"","sources":["contradiction.ts"],"names":[],"mappings":"AASA;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAsB;IAEtB,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACtB,IAAI,IAAI,CAAC,SAAS,KAAK,YAAY,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC9D,OAAO;gBACL,IAAI;gBACJ,mBAAmB,EAAE,QAAiB;gBACtC,aAAa,EAAE,IAAI,CAAC,iBAAiB;aACtC,CAAC;QACJ,CAAC;QACD,OAAO;YACL,IAAI;YACJ,mBAAmB,EAAE,MAAe;YACpC,aAAa,EAAE,IAAI;SACpB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cross-Fact Edge Linker — Intelligent Relationship Detection
|
|
3
|
-
*
|
|
4
|
-
* After extraction, finds existing facts that share entities with new facts
|
|
5
|
-
* AND are semantically similar (not just any shared entity). Creates
|
|
6
|
-
* 'relates_to' edges only when facts are actually about the same topic.
|
|
7
|
-
*
|
|
8
|
-
* Heuristics:
|
|
9
|
-
* 1. Shared entity (required) — both facts mention the same entity
|
|
10
|
-
* 2. Content similarity (required) — facts have overlapping keywords (>30% overlap)
|
|
11
|
-
* 3. Temporal proximity (bonus) — facts created within 7 days get priority
|
|
12
|
-
* 4. Skip generic entities — "user" entity is too broad to link on
|
|
13
|
-
*/
|
|
14
|
-
import type { StorageAdapter } from '../adapters/storage.js';
|
|
15
|
-
/**
|
|
16
|
-
* Link newly created facts to existing related facts through shared entities.
|
|
17
|
-
* Uses keyword overlap + temporal proximity to determine relevance.
|
|
18
|
-
*
|
|
19
|
-
* Only creates edges when facts are genuinely related — not just because
|
|
20
|
-
* they both mention "Steno" or "user".
|
|
21
|
-
*/
|
|
22
|
-
export declare function linkRelatedFacts(storage: StorageAdapter, tenantId: string, newFactIds: string[], entityIdMap: Map<string, string>): Promise<number>;
|
|
23
|
-
//# sourceMappingURL=cross-linker.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cross-linker.d.ts","sourceRoot":"","sources":["cross-linker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAyD7D;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAAE,EACpB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,MAAM,CAAC,CAkFjB"}
|
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cross-Fact Edge Linker — Intelligent Relationship Detection
|
|
3
|
-
*
|
|
4
|
-
* After extraction, finds existing facts that share entities with new facts
|
|
5
|
-
* AND are semantically similar (not just any shared entity). Creates
|
|
6
|
-
* 'relates_to' edges only when facts are actually about the same topic.
|
|
7
|
-
*
|
|
8
|
-
* Heuristics:
|
|
9
|
-
* 1. Shared entity (required) — both facts mention the same entity
|
|
10
|
-
* 2. Content similarity (required) — facts have overlapping keywords (>30% overlap)
|
|
11
|
-
* 3. Temporal proximity (bonus) — facts created within 7 days get priority
|
|
12
|
-
* 4. Skip generic entities — "user" entity is too broad to link on
|
|
13
|
-
*/
|
|
14
|
-
const GENERIC_ENTITIES = new Set(['user', 'assistant', 'navia', 'steno']);
|
|
15
|
-
const MIN_KEYWORD_OVERLAP = 0.25; // 25% of keywords must overlap
|
|
16
|
-
const MAX_LINKS_PER_FACT = 3; // Don't create too many edges per new fact
|
|
17
|
-
const TEMPORAL_WINDOW_DAYS = 14; // Prefer facts within 2 weeks
|
|
18
|
-
/**
|
|
19
|
-
* Extract meaningful keywords from fact content for similarity comparison.
|
|
20
|
-
* Strips stop words, lowercases, returns unique tokens.
|
|
21
|
-
*/
|
|
22
|
-
function extractKeywords(content) {
|
|
23
|
-
const stopWords = new Set([
|
|
24
|
-
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
25
|
-
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
|
|
26
|
-
'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
|
|
27
|
-
'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
|
|
28
|
-
'before', 'after', 'above', 'below', 'between', 'and', 'but', 'or',
|
|
29
|
-
'not', 'no', 'nor', 'so', 'yet', 'both', 'each', 'few', 'more',
|
|
30
|
-
'most', 'other', 'some', 'such', 'than', 'too', 'very', 'just',
|
|
31
|
-
'about', 'up', 'out', 'if', 'then', 'that', 'this', 'these', 'those',
|
|
32
|
-
'it', 'its', 'user', 'users', 'they', 'them', 'their', 'he', 'she',
|
|
33
|
-
]);
|
|
34
|
-
return new Set(content.toLowerCase()
|
|
35
|
-
.replace(/[^a-z0-9\s-]/g, ' ')
|
|
36
|
-
.split(/\s+/)
|
|
37
|
-
.filter(w => w.length >= 3 && !stopWords.has(w)));
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Calculate keyword overlap ratio between two facts.
|
|
41
|
-
* Returns 0-1 where 1 = identical keywords.
|
|
42
|
-
*/
|
|
43
|
-
function keywordOverlap(a, b) {
|
|
44
|
-
if (a.size === 0 || b.size === 0)
|
|
45
|
-
return 0;
|
|
46
|
-
let shared = 0;
|
|
47
|
-
for (const word of a) {
|
|
48
|
-
if (b.has(word))
|
|
49
|
-
shared++;
|
|
50
|
-
}
|
|
51
|
-
const minSize = Math.min(a.size, b.size);
|
|
52
|
-
return shared / minSize;
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* Check if two facts are temporally close (within TEMPORAL_WINDOW_DAYS).
|
|
56
|
-
*/
|
|
57
|
-
function isTemporallyClose(a, b) {
|
|
58
|
-
const aDate = new Date(a.createdAt).getTime();
|
|
59
|
-
const bDate = new Date(b.createdAt).getTime();
|
|
60
|
-
const diffDays = Math.abs(aDate - bDate) / (1000 * 60 * 60 * 24);
|
|
61
|
-
return diffDays <= TEMPORAL_WINDOW_DAYS;
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Link newly created facts to existing related facts through shared entities.
|
|
65
|
-
* Uses keyword overlap + temporal proximity to determine relevance.
|
|
66
|
-
*
|
|
67
|
-
* Only creates edges when facts are genuinely related — not just because
|
|
68
|
-
* they both mention "Steno" or "user".
|
|
69
|
-
*/
|
|
70
|
-
export async function linkRelatedFacts(storage, tenantId, newFactIds, entityIdMap) {
|
|
71
|
-
if (newFactIds.length === 0 || entityIdMap.size === 0)
|
|
72
|
-
return 0;
|
|
73
|
-
let edgesCreated = 0;
|
|
74
|
-
// Get the new facts' content for keyword extraction
|
|
75
|
-
const newFacts = await storage.getFactsByIds(tenantId, newFactIds);
|
|
76
|
-
if (newFacts.length === 0)
|
|
77
|
-
return 0;
|
|
78
|
-
const newFactKeywords = new Map();
|
|
79
|
-
for (const f of newFacts) {
|
|
80
|
-
newFactKeywords.set(f.id, extractKeywords(f.content));
|
|
81
|
-
}
|
|
82
|
-
// For each NON-GENERIC entity, find existing facts that also mention it
|
|
83
|
-
for (const [canonicalName, entityId] of entityIdMap) {
|
|
84
|
-
if (GENERIC_ENTITIES.has(canonicalName))
|
|
85
|
-
continue; // Skip "user", "navia", etc.
|
|
86
|
-
try {
|
|
87
|
-
const linkedFacts = await storage.getFactsForEntity(tenantId, entityId, { limit: 20 });
|
|
88
|
-
if (linkedFacts.data.length < 2)
|
|
89
|
-
continue;
|
|
90
|
-
const newFactSet = new Set(newFactIds);
|
|
91
|
-
const existingFacts = linkedFacts.data.filter(f => !newFactSet.has(f.id) &&
|
|
92
|
-
!f.tags?.includes('scratchpad') // Skip scratchpad blobs
|
|
93
|
-
);
|
|
94
|
-
for (const newFact of newFacts) {
|
|
95
|
-
const newKw = newFactKeywords.get(newFact.id);
|
|
96
|
-
if (!newKw || newKw.size < 2)
|
|
97
|
-
continue; // Too short to compare
|
|
98
|
-
let linksForThisFact = 0;
|
|
99
|
-
for (const existingFact of existingFacts) {
|
|
100
|
-
if (linksForThisFact >= MAX_LINKS_PER_FACT)
|
|
101
|
-
break;
|
|
102
|
-
const existingKw = extractKeywords(existingFact.content);
|
|
103
|
-
const overlap = keywordOverlap(newKw, existingKw);
|
|
104
|
-
// Must have meaningful keyword overlap
|
|
105
|
-
if (overlap < MIN_KEYWORD_OVERLAP)
|
|
106
|
-
continue;
|
|
107
|
-
// Boost score for temporal proximity
|
|
108
|
-
const temporalBoost = isTemporallyClose(newFact, existingFact) ? 0.1 : 0;
|
|
109
|
-
const relevanceScore = overlap + temporalBoost;
|
|
110
|
-
try {
|
|
111
|
-
await storage.createEdge({
|
|
112
|
-
id: crypto.randomUUID(),
|
|
113
|
-
tenantId,
|
|
114
|
-
sourceId: entityId,
|
|
115
|
-
targetId: entityId,
|
|
116
|
-
relation: 'relates_to',
|
|
117
|
-
edgeType: 'associative',
|
|
118
|
-
weight: Math.min(relevanceScore, 1.0),
|
|
119
|
-
factId: newFact.id,
|
|
120
|
-
confidence: overlap,
|
|
121
|
-
metadata: {
|
|
122
|
-
linkedFactId: existingFact.id,
|
|
123
|
-
reason: 'shared_entity_and_content',
|
|
124
|
-
entity: canonicalName,
|
|
125
|
-
keywordOverlap: overlap.toFixed(2),
|
|
126
|
-
},
|
|
127
|
-
});
|
|
128
|
-
edgesCreated++;
|
|
129
|
-
linksForThisFact++;
|
|
130
|
-
}
|
|
131
|
-
catch {
|
|
132
|
-
// Duplicate or constraint — skip
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
catch {
|
|
138
|
-
// Entity lookup failed — continue
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
if (edgesCreated > 0) {
|
|
142
|
-
console.error(`[steno] Cross-linked ${edgesCreated} related facts via shared entities`);
|
|
143
|
-
}
|
|
144
|
-
return edgesCreated;
|
|
145
|
-
}
|
|
146
|
-
//# sourceMappingURL=cross-linker.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cross-linker.js","sourceRoot":"","sources":["cross-linker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAC1E,MAAM,mBAAmB,GAAG,IAAI,CAAC,CAAC,+BAA+B;AACjE,MAAM,kBAAkB,GAAG,CAAC,CAAC,CAAC,2CAA2C;AACzE,MAAM,oBAAoB,GAAG,EAAE,CAAC,CAAC,8BAA8B;AAE/D;;;GAGG;AACH,SAAS,eAAe,CAAC,OAAe;IACtC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;QACxB,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;QACnE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;QACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;QACjE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ;QACnE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI;QAClE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;QAC9D,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;QAC9D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;QACpE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK;KACnE,CAAC,CAAC;IAEH,OAAO,IAAI,GAAG,CACZ,OAAO,CAAC,WAAW,EAAE;SAClB,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC;SAC7B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CACnD,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,CAAc,EAAE,CAAc;IACpD,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,EAAE,CAAC;IAC5B,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,MAAM,GAAG,OAAO,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,CAAO,EAAE,CAAO;IACzC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IACjE,OAAO,QAAQ,IAAI,oBAAoB,CAAC;AAC1C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAuB,EACvB,QAAgB,EAChB,UAAoB,EACpB,WAAgC;IAEhC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,oDAAoD;IACpD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACnE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEpC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,wEAAwE;IACxE,KAAK,MAAM,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACpD,IAAI,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC;YAAE,SAAS,CAAC,6BAA6B;QAEhF,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;YACvF,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAE1C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;YACvC,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAChD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrB,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC,wBAAwB;aACzD,CAAC;YAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC9C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC;oBAAE,SAAS,CAAC,uBAAuB;gBAE/D,IAAI,gBAAgB,GAAG,CAAC,CAAC;gBAEzB,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;oBACzC,IAAI,gBAAgB,IAAI,kBAAkB;wBAAE,MAAM;oBAElD,MAAM,UAAU,GAAG,eAAe,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBACzD,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;oBAElD,uCAAuC;oBACvC,IAAI,OAAO,GAAG,mBAAmB;wBAAE,SAAS;oBAE5C,qCAAqC;oBACrC,MAAM,aAAa,GAAG,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzE,MAAM,cAAc,GAAG,OAAO,GAAG,aAAa,CAAC;oBAE/C,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,UAAU,CAAC;4BACvB,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;4BACvB,QAAQ;4BACR,QAAQ,EAAE,QAAQ;4BAClB,QAAQ,EAAE,QAAQ;4BAClB,QAAQ,EAAE,YAAY;4BACtB,QAAQ,EAAE,aAAa;4BACvB,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,CAAC;4BACrC,MAAM,EAAE,OAAO,CAAC,EAAE;4BAClB,UAAU,EAAE,OAAO;4BACnB,QAAQ,EAAE;gCACR,YAAY,EAAE,YAAY,CAAC,EAAE;gCAC7B,MAAM,EAAE,2BAA2B;gCACnC,MAAM,EAAE,aAAa;gCACrB,cAAc,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;6BACnC;yBACF,CAAC,CAAC;wBACH,YAAY,EAAE,CAAC;wBACf,gBAAgB,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC;wBACP,iCAAiC;oBACnC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,KAAK,CAAC,wBAAwB,YAAY,oCAAoC,CAAC,CAAC;IAC1F,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { EmbeddingAdapter } from '../adapters/embedding.js';
|
|
2
|
-
import type { LLMAdapter } from '../adapters/llm.js';
|
|
3
|
-
import type { StorageAdapter } from '../adapters/storage.js';
|
|
4
|
-
import type { ExtractedFact } from './types.js';
|
|
5
|
-
export interface DedupConfig {
|
|
6
|
-
storage: StorageAdapter;
|
|
7
|
-
embedding: EmbeddingAdapter;
|
|
8
|
-
llm: LLMAdapter;
|
|
9
|
-
similarityThreshold?: number;
|
|
10
|
-
}
|
|
11
|
-
export declare function deduplicateFacts(config: DedupConfig, facts: ExtractedFact[], tenantId: string, scope: string, scopeId: string): Promise<ExtractedFact[]>;
|
|
12
|
-
//# sourceMappingURL=dedup.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["dedup.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,UAAU,EAAc,MAAM,oBAAoB,CAAC;AACjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,cAAc,CAAC;IACxB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,GAAG,EAAE,UAAU,CAAC;IAChB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,aAAa,EAAE,EACtB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,aAAa,EAAE,CAAC,CA+C1B"}
|
package/src/extraction/dedup.js
DELETED
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
export async function deduplicateFacts(config, facts, tenantId, scope, scopeId) {
|
|
2
|
-
const threshold = config.similarityThreshold ?? 0.55;
|
|
3
|
-
const result = [];
|
|
4
|
-
for (const fact of facts) {
|
|
5
|
-
// If LLM already decided and it's not 'add', trust it
|
|
6
|
-
if (fact.operation && fact.operation !== 'add') {
|
|
7
|
-
result.push(fact);
|
|
8
|
-
continue;
|
|
9
|
-
}
|
|
10
|
-
// Embed the fact
|
|
11
|
-
const embedding = await config.embedding.embed(fact.content);
|
|
12
|
-
// Search for similar existing facts
|
|
13
|
-
const matches = await config.storage.vectorSearch({
|
|
14
|
-
embedding,
|
|
15
|
-
tenantId,
|
|
16
|
-
scope,
|
|
17
|
-
scopeId,
|
|
18
|
-
limit: 5,
|
|
19
|
-
minSimilarity: threshold,
|
|
20
|
-
validOnly: true,
|
|
21
|
-
});
|
|
22
|
-
if (matches.length === 0) {
|
|
23
|
-
// No similar facts — this is new
|
|
24
|
-
result.push({ ...fact, operation: 'add' });
|
|
25
|
-
continue;
|
|
26
|
-
}
|
|
27
|
-
// Similar facts found — ask LLM to classify
|
|
28
|
-
const decision = await classifyWithLLM(config.llm, fact.content, matches.map(m => ({
|
|
29
|
-
id: m.fact.id,
|
|
30
|
-
lineageId: m.fact.lineageId ?? '',
|
|
31
|
-
content: m.fact.content,
|
|
32
|
-
similarity: m.similarity,
|
|
33
|
-
})));
|
|
34
|
-
result.push({ ...fact, ...decision });
|
|
35
|
-
}
|
|
36
|
-
return result;
|
|
37
|
-
}
|
|
38
|
-
async function classifyWithLLM(llm, candidateContent, existingMatches) {
|
|
39
|
-
const messages = [
|
|
40
|
-
{
|
|
41
|
-
role: 'system',
|
|
42
|
-
content: `You are a memory deduplication classifier. Compare a NEW fact against EXISTING facts.
|
|
43
|
-
|
|
44
|
-
RULES:
|
|
45
|
-
- "noop" if the new fact says the SAME thing as an existing fact (even with different wording). Be AGGRESSIVE about noop — "User's name is Caroline" and "User is called Caroline" are NOOP.
|
|
46
|
-
- "update" if the new fact adds detail or changes a value ("User likes cats" → "User loves cats and has 3")
|
|
47
|
-
- "contradict" if the new fact directly conflicts ("User likes cats" vs "User hates cats")
|
|
48
|
-
- "add" ONLY if genuinely new information not covered by ANY existing fact
|
|
49
|
-
|
|
50
|
-
Return JSON:
|
|
51
|
-
{"operation": "add|update|noop|contradict", "existing_lineage_id": "...", "contradicts_fact_id": "..."}
|
|
52
|
-
|
|
53
|
-
Return ONLY valid JSON.`,
|
|
54
|
-
},
|
|
55
|
-
{
|
|
56
|
-
role: 'user',
|
|
57
|
-
content: `NEW FACT: "${candidateContent}"
|
|
58
|
-
|
|
59
|
-
EXISTING FACTS:
|
|
60
|
-
${existingMatches.map(m => `[id: ${m.id}, lineage_id: ${m.lineageId}, similarity: ${m.similarity.toFixed(3)}] ${m.content}`).join('\n')}
|
|
61
|
-
|
|
62
|
-
Classify the new fact.`,
|
|
63
|
-
},
|
|
64
|
-
];
|
|
65
|
-
try {
|
|
66
|
-
const response = await llm.complete(messages, { temperature: 0, responseFormat: 'json' });
|
|
67
|
-
const parsed = JSON.parse(response.content);
|
|
68
|
-
const operation = isValidDedupOp(parsed.operation) ? parsed.operation : 'add';
|
|
69
|
-
// Map dedup operations to relational versioning types
|
|
70
|
-
const relationType = operation === 'update' ? 'updates'
|
|
71
|
-
: operation === 'contradict' ? 'updates'
|
|
72
|
-
: undefined;
|
|
73
|
-
// Find the best matching existing fact for the relation
|
|
74
|
-
const relatedFactId = (operation === 'update' || operation === 'contradict') && existingMatches.length > 0
|
|
75
|
-
? existingMatches[0].id
|
|
76
|
-
: undefined;
|
|
77
|
-
return {
|
|
78
|
-
operation,
|
|
79
|
-
existingLineageId: typeof parsed.existing_lineage_id === 'string' ? parsed.existing_lineage_id : undefined,
|
|
80
|
-
contradictsFactId: typeof parsed.contradicts_fact_id === 'string' ? parsed.contradicts_fact_id : undefined,
|
|
81
|
-
relationType,
|
|
82
|
-
relatedFactId,
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
catch {
|
|
86
|
-
// If LLM fails, default to 'add' (safe — might create a duplicate, but won't lose data)
|
|
87
|
-
return { operation: 'add' };
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
function isValidDedupOp(op) {
|
|
91
|
-
return typeof op === 'string' && ['add', 'update', 'noop', 'contradict', 'invalidate'].includes(op);
|
|
92
|
-
}
|
|
93
|
-
//# sourceMappingURL=dedup.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["dedup.ts"],"names":[],"mappings":"AAYA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAmB,EACnB,KAAsB,EACtB,QAAgB,EAChB,KAAa,EACb,OAAe;IAEf,MAAM,SAAS,GAAG,MAAM,CAAC,mBAAmB,IAAI,IAAI,CAAC;IACrD,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,sDAAsD;QACtD,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC/C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QAED,iBAAiB;QACjB,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE7D,oCAAoC;QACpC,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;YAChD,SAAS;YACT,QAAQ;YACR,KAAK;YACL,OAAO;YACP,KAAK,EAAE,CAAC;YACR,aAAa,EAAE,SAAS;YACxB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,iCAAiC;YACjC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3C,SAAS;QACX,CAAC;QAED,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,MAAM,eAAe,CACpC,MAAM,CAAC,GAAG,EACV,IAAI,CAAC,OAAO,EACZ,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChB,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE;YACb,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE;YACjC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO;YACvB,UAAU,EAAE,CAAC,CAAC,UAAU;SACzB,CAAC,CAAC,CACJ,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,GAAe,EACf,gBAAwB,EACxB,eAA8F;IAE9F,MAAM,QAAQ,GAAiB;QAC7B;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE;;;;;;;;;;;wBAWS;SACnB;QACD;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,cAAc,gBAAgB;;;EAG3C,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,SAAS,iBAAiB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;uBAEhH;SAClB;KACF,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;QAC1F,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAA4B,CAAC;QAEvE,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QAE9E,sDAAsD;QACtD,MAAM,YAAY,GAAG,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAkB;YAC9D,CAAC,CAAC,SAAS,KAAK,YAAY,CAAC,CAAC,CAAC,SAAkB;gBACjD,CAAC,CAAC,SAAS,CAAC;QAEd,wDAAwD;QACxD,MAAM,aAAa,GAAG,CAAC,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,YAAY,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC;YACxG,CAAC,CAAC,eAAe,CAAC,CAAC,CAAE,CAAC,EAAE;YACxB,CAAC,CAAC,SAAS,CAAC;QAEd,OAAO;YACL,SAAS;YACT,iBAAiB,EACf,OAAO,MAAM,CAAC,mBAAmB,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YACzF,iBAAiB,EACf,OAAO,MAAM,CAAC,mBAAmB,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YACzF,YAAY;YACZ,aAAa;SACd,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,wFAAwF;QACxF,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,EAAW;IACjC,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACtG,CAAC"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import type { StorageAdapter } from '../adapters/storage.js';
|
|
2
|
-
import type { EmbeddingAdapter } from '../adapters/embedding.js';
|
|
3
|
-
import type { ExtractedEntity, ExtractedEdge } from './types.js';
|
|
4
|
-
export interface EntityPersistenceResult {
|
|
5
|
-
entitiesCreated: number;
|
|
6
|
-
edgesCreated: number;
|
|
7
|
-
entityIdMap: Map<string, string>;
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* Create or find all entities in the database.
|
|
11
|
-
* Does NOT link entities to any fact or create edges.
|
|
12
|
-
* Returns a map of canonicalName → entity.id for use in subsequent operations.
|
|
13
|
-
*/
|
|
14
|
-
export declare function buildEntityIdMap(storage: StorageAdapter, embedding: EmbeddingAdapter, tenantId: string, entities: ExtractedEntity[]): Promise<{
|
|
15
|
-
entityIdMap: Map<string, string>;
|
|
16
|
-
entitiesCreated: number;
|
|
17
|
-
}>;
|
|
18
|
-
export declare function persistEdges(storage: StorageAdapter, tenantId: string, factId: string, edges: ExtractedEdge[], entityIdMap: Map<string, string>): Promise<number>;
|
|
19
|
-
/**
|
|
20
|
-
* Persist extracted entities and edges to the database.
|
|
21
|
-
* - Creates new entities if canonical name doesn't exist for this tenant
|
|
22
|
-
* - Reuses existing entities if canonical name matches
|
|
23
|
-
* - Links all entities to the fact via fact_entities junction
|
|
24
|
-
* - Creates edges between entities
|
|
25
|
-
*
|
|
26
|
-
* @deprecated Use buildEntityIdMap + persistEdges directly for better control.
|
|
27
|
-
* This function is kept for backward compatibility.
|
|
28
|
-
*/
|
|
29
|
-
export declare function persistEntitiesAndEdges(storage: StorageAdapter, embedding: EmbeddingAdapter, tenantId: string, factId: string, entities: ExtractedEntity[], edges: ExtractedEdge[]): Promise<EntityPersistenceResult>;
|
|
30
|
-
//# sourceMappingURL=entity-extractor.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"entity-extractor.d.ts","sourceRoot":"","sources":["entity-extractor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEjE,MAAM,WAAW,uBAAuB;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,gBAAgB,EAC3B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,eAAe,EAAE,GAC1B,OAAO,CAAC;IAAE,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,eAAe,EAAE,MAAM,CAAA;CAAE,CAAC,CAuCxE;AA8CD,wBAAsB,YAAY,CAChC,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,EAAE,EACtB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC/B,OAAO,CAAC,MAAM,CAAC,CA6CjB;AAED;;;;;;;;;GASG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,cAAc,EACvB,SAAS,EAAE,gBAAgB,EAC3B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,eAAe,EAAE,EAC3B,KAAK,EAAE,aAAa,EAAE,GACrB,OAAO,CAAC,uBAAuB,CAAC,CAiBlC"}
|
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Create or find all entities in the database.
|
|
3
|
-
* Does NOT link entities to any fact or create edges.
|
|
4
|
-
* Returns a map of canonicalName → entity.id for use in subsequent operations.
|
|
5
|
-
*/
|
|
6
|
-
export async function buildEntityIdMap(storage, embedding, tenantId, entities) {
|
|
7
|
-
const entityIdMap = new Map();
|
|
8
|
-
let entitiesCreated = 0;
|
|
9
|
-
// Deduplicate entities by canonical name within this batch
|
|
10
|
-
const uniqueEntities = new Map();
|
|
11
|
-
for (const entity of entities) {
|
|
12
|
-
if (!uniqueEntities.has(entity.canonicalName)) {
|
|
13
|
-
uniqueEntities.set(entity.canonicalName, entity);
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
// Create or find each entity
|
|
17
|
-
for (const entity of uniqueEntities.values()) {
|
|
18
|
-
const existing = await storage.findEntityByCanonicalName(tenantId, entity.canonicalName, entity.entityType);
|
|
19
|
-
if (existing) {
|
|
20
|
-
entityIdMap.set(entity.canonicalName, existing.id);
|
|
21
|
-
}
|
|
22
|
-
else {
|
|
23
|
-
const id = crypto.randomUUID();
|
|
24
|
-
const emb = await embedding.embed(entity.name);
|
|
25
|
-
await storage.createEntity({
|
|
26
|
-
...entity,
|
|
27
|
-
id,
|
|
28
|
-
tenantId,
|
|
29
|
-
embedding: emb,
|
|
30
|
-
embeddingModel: embedding.model,
|
|
31
|
-
embeddingDim: embedding.dimensions,
|
|
32
|
-
});
|
|
33
|
-
entityIdMap.set(entity.canonicalName, id);
|
|
34
|
-
entitiesCreated++;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
return { entityIdMap, entitiesCreated };
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Create edges between already-persisted entities.
|
|
41
|
-
* Uses the entityIdMap built by buildEntityIdMap.
|
|
42
|
-
*/
|
|
43
|
-
// Normalize relation names so "loves_deeply" and "loves" merge
|
|
44
|
-
const RELATION_SYNONYMS = {
|
|
45
|
-
loves_deeply: 'loves',
|
|
46
|
-
has_relationship: 'partner_of',
|
|
47
|
-
has_relationship_with: 'partner_of',
|
|
48
|
-
dating: 'partner_of',
|
|
49
|
-
in_relationship_with: 'partner_of',
|
|
50
|
-
romantic_partner: 'partner_of',
|
|
51
|
-
girlfriend_of: 'partner_of',
|
|
52
|
-
boyfriend_of: 'partner_of',
|
|
53
|
-
married_to: 'partner_of',
|
|
54
|
-
works_for: 'works_at',
|
|
55
|
-
employed_at: 'works_at',
|
|
56
|
-
employed_by: 'works_at',
|
|
57
|
-
resides_in: 'lives_in',
|
|
58
|
-
located_at: 'located_in',
|
|
59
|
-
friends_with: 'friend_of',
|
|
60
|
-
acquainted_with: 'knows',
|
|
61
|
-
knows_about: 'knows',
|
|
62
|
-
interested_in: 'prefers',
|
|
63
|
-
attracted_to: 'prefers',
|
|
64
|
-
likes: 'prefers',
|
|
65
|
-
replaces: 'updates',
|
|
66
|
-
supersedes: 'updates',
|
|
67
|
-
overrides: 'updates',
|
|
68
|
-
corrects: 'updates',
|
|
69
|
-
supplements: 'extends',
|
|
70
|
-
refines: 'extends',
|
|
71
|
-
elaborates: 'extends',
|
|
72
|
-
details: 'extends',
|
|
73
|
-
infers: 'derives',
|
|
74
|
-
implies: 'derives',
|
|
75
|
-
combines: 'derives',
|
|
76
|
-
};
|
|
77
|
-
function normalizeRelation(relation) {
|
|
78
|
-
const lower = relation.toLowerCase().trim();
|
|
79
|
-
return RELATION_SYNONYMS[lower] ?? lower;
|
|
80
|
-
}
|
|
81
|
-
export async function persistEdges(storage, tenantId, factId, edges, entityIdMap) {
|
|
82
|
-
// Deduplicate edges by (source, target, normalized_relation) within this batch
|
|
83
|
-
const seen = new Set();
|
|
84
|
-
const dedupedEdges = [];
|
|
85
|
-
for (const edge of edges) {
|
|
86
|
-
const normalizedRelation = normalizeRelation(edge.relation);
|
|
87
|
-
const key = `${edge.sourceName}|${normalizedRelation}|${edge.targetName}`;
|
|
88
|
-
if (!seen.has(key)) {
|
|
89
|
-
seen.add(key);
|
|
90
|
-
dedupedEdges.push({ ...edge, relation: normalizedRelation });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
let edgesCreated = 0;
|
|
94
|
-
for (const edge of dedupedEdges) {
|
|
95
|
-
const sourceId = entityIdMap.get(edge.sourceName);
|
|
96
|
-
const targetId = entityIdMap.get(edge.targetName);
|
|
97
|
-
if (sourceId && targetId) {
|
|
98
|
-
try {
|
|
99
|
-
await storage.createEdge({
|
|
100
|
-
tenantId,
|
|
101
|
-
sourceId,
|
|
102
|
-
targetId,
|
|
103
|
-
relation: edge.relation,
|
|
104
|
-
edgeType: edge.edgeType,
|
|
105
|
-
confidence: edge.confidence,
|
|
106
|
-
weight: 1.0,
|
|
107
|
-
metadata: {},
|
|
108
|
-
factId,
|
|
109
|
-
id: crypto.randomUUID(),
|
|
110
|
-
});
|
|
111
|
-
edgesCreated++;
|
|
112
|
-
}
|
|
113
|
-
catch {
|
|
114
|
-
// Edge creation failed (e.g., duplicate) — continue
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
if (!sourceId || !targetId) {
|
|
118
|
-
console.warn(`[steno] Edge dropped: "${edge.sourceName}" → "${edge.relation}" → "${edge.targetName}" ` +
|
|
119
|
-
`(source=${sourceId ? 'found' : 'MISSING'}, target=${targetId ? 'found' : 'MISSING'}) ` +
|
|
120
|
-
`entityMap keys: [${[...entityIdMap.keys()].join(', ')}]`);
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
return edgesCreated;
|
|
124
|
-
}
|
|
125
|
-
/**
|
|
126
|
-
* Persist extracted entities and edges to the database.
|
|
127
|
-
* - Creates new entities if canonical name doesn't exist for this tenant
|
|
128
|
-
* - Reuses existing entities if canonical name matches
|
|
129
|
-
* - Links all entities to the fact via fact_entities junction
|
|
130
|
-
* - Creates edges between entities
|
|
131
|
-
*
|
|
132
|
-
* @deprecated Use buildEntityIdMap + persistEdges directly for better control.
|
|
133
|
-
* This function is kept for backward compatibility.
|
|
134
|
-
*/
|
|
135
|
-
export async function persistEntitiesAndEdges(storage, embedding, tenantId, factId, entities, edges) {
|
|
136
|
-
const { entityIdMap, entitiesCreated } = await buildEntityIdMap(storage, embedding, tenantId, entities);
|
|
137
|
-
// Link all entities to this fact
|
|
138
|
-
for (const [canonicalName, entityId] of entityIdMap) {
|
|
139
|
-
void canonicalName; // used as key only
|
|
140
|
-
await storage.linkFactEntity(factId, entityId, 'mentioned');
|
|
141
|
-
}
|
|
142
|
-
const edgesCreated = await persistEdges(storage, tenantId, factId, edges, entityIdMap);
|
|
143
|
-
return { entitiesCreated, edgesCreated, entityIdMap };
|
|
144
|
-
}
|
|
145
|
-
//# sourceMappingURL=entity-extractor.js.map
|