ai-memory-layer 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/LICENSE +21 -0
- package/README.md +765 -0
- package/bin/memory-server.mjs +157 -0
- package/dist/adapters/memory/embeddings.d.ts +4 -0
- package/dist/adapters/memory/embeddings.d.ts.map +1 -0
- package/dist/adapters/memory/embeddings.js +53 -0
- package/dist/adapters/memory/embeddings.js.map +1 -0
- package/dist/adapters/memory/index.d.ts +7 -0
- package/dist/adapters/memory/index.d.ts.map +1 -0
- package/dist/adapters/memory/index.js +650 -0
- package/dist/adapters/memory/index.js.map +1 -0
- package/dist/adapters/postgres/index.d.ts +38 -0
- package/dist/adapters/postgres/index.d.ts.map +1 -0
- package/dist/adapters/postgres/index.js +982 -0
- package/dist/adapters/postgres/index.js.map +1 -0
- package/dist/adapters/sqlite/embeddings.d.ts +5 -0
- package/dist/adapters/sqlite/embeddings.d.ts.map +1 -0
- package/dist/adapters/sqlite/embeddings.js +122 -0
- package/dist/adapters/sqlite/embeddings.js.map +1 -0
- package/dist/adapters/sqlite/index.d.ts +8 -0
- package/dist/adapters/sqlite/index.d.ts.map +1 -0
- package/dist/adapters/sqlite/index.js +839 -0
- package/dist/adapters/sqlite/index.js.map +1 -0
- package/dist/adapters/sqlite/mappers.d.ts +40 -0
- package/dist/adapters/sqlite/mappers.d.ts.map +1 -0
- package/dist/adapters/sqlite/mappers.js +95 -0
- package/dist/adapters/sqlite/mappers.js.map +1 -0
- package/dist/adapters/sqlite/schema.d.ts +4 -0
- package/dist/adapters/sqlite/schema.d.ts.map +1 -0
- package/dist/adapters/sqlite/schema.js +394 -0
- package/dist/adapters/sqlite/schema.js.map +1 -0
- package/dist/adapters/sync-to-async.d.ts +15 -0
- package/dist/adapters/sync-to-async.d.ts.map +1 -0
- package/dist/adapters/sync-to-async.js +95 -0
- package/dist/adapters/sync-to-async.js.map +1 -0
- package/dist/cli/inspect.d.ts +34 -0
- package/dist/cli/inspect.d.ts.map +1 -0
- package/dist/cli/inspect.js +190 -0
- package/dist/cli/inspect.js.map +1 -0
- package/dist/contracts/async-storage.d.ts +86 -0
- package/dist/contracts/async-storage.d.ts.map +1 -0
- package/dist/contracts/async-storage.js +2 -0
- package/dist/contracts/async-storage.js.map +1 -0
- package/dist/contracts/embedding.d.ts +22 -0
- package/dist/contracts/embedding.d.ts.map +1 -0
- package/dist/contracts/embedding.js +2 -0
- package/dist/contracts/embedding.js.map +1 -0
- package/dist/contracts/identity.d.ts +29 -0
- package/dist/contracts/identity.d.ts.map +1 -0
- package/dist/contracts/identity.js +34 -0
- package/dist/contracts/identity.js.map +1 -0
- package/dist/contracts/observability.d.ts +18 -0
- package/dist/contracts/observability.d.ts.map +1 -0
- package/dist/contracts/observability.js +7 -0
- package/dist/contracts/observability.js.map +1 -0
- package/dist/contracts/policy.d.ts +108 -0
- package/dist/contracts/policy.d.ts.map +1 -0
- package/dist/contracts/policy.js +107 -0
- package/dist/contracts/policy.js.map +1 -0
- package/dist/contracts/storage.d.ts +78 -0
- package/dist/contracts/storage.d.ts.map +1 -0
- package/dist/contracts/storage.js +2 -0
- package/dist/contracts/storage.js.map +1 -0
- package/dist/contracts/types.d.ts +381 -0
- package/dist/contracts/types.d.ts.map +1 -0
- package/dist/contracts/types.js +94 -0
- package/dist/contracts/types.js.map +1 -0
- package/dist/core/circuit-breaker.d.ts +11 -0
- package/dist/core/circuit-breaker.d.ts.map +1 -0
- package/dist/core/circuit-breaker.js +38 -0
- package/dist/core/circuit-breaker.js.map +1 -0
- package/dist/core/context.d.ts +56 -0
- package/dist/core/context.d.ts.map +1 -0
- package/dist/core/context.js +345 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/events.d.ts +8 -0
- package/dist/core/events.d.ts.map +1 -0
- package/dist/core/events.js +25 -0
- package/dist/core/events.js.map +1 -0
- package/dist/core/extractor.d.ts +37 -0
- package/dist/core/extractor.d.ts.map +1 -0
- package/dist/core/extractor.js +448 -0
- package/dist/core/extractor.js.map +1 -0
- package/dist/core/formatter.d.ts +25 -0
- package/dist/core/formatter.d.ts.map +1 -0
- package/dist/core/formatter.js +97 -0
- package/dist/core/formatter.js.map +1 -0
- package/dist/core/knowledge-lifecycle.d.ts +15 -0
- package/dist/core/knowledge-lifecycle.d.ts.map +1 -0
- package/dist/core/knowledge-lifecycle.js +103 -0
- package/dist/core/knowledge-lifecycle.js.map +1 -0
- package/dist/core/maintenance.d.ts +13 -0
- package/dist/core/maintenance.d.ts.map +1 -0
- package/dist/core/maintenance.js +102 -0
- package/dist/core/maintenance.js.map +1 -0
- package/dist/core/manager.d.ts +110 -0
- package/dist/core/manager.d.ts.map +1 -0
- package/dist/core/manager.js +640 -0
- package/dist/core/manager.js.map +1 -0
- package/dist/core/monitor.d.ts +73 -0
- package/dist/core/monitor.d.ts.map +1 -0
- package/dist/core/monitor.js +395 -0
- package/dist/core/monitor.js.map +1 -0
- package/dist/core/orchestrator.d.ts +64 -0
- package/dist/core/orchestrator.d.ts.map +1 -0
- package/dist/core/orchestrator.js +916 -0
- package/dist/core/orchestrator.js.map +1 -0
- package/dist/core/presets.d.ts +15 -0
- package/dist/core/presets.d.ts.map +1 -0
- package/dist/core/presets.js +99 -0
- package/dist/core/presets.js.map +1 -0
- package/dist/core/provider-managers.d.ts +47 -0
- package/dist/core/provider-managers.d.ts.map +1 -0
- package/dist/core/provider-managers.js +112 -0
- package/dist/core/provider-managers.js.map +1 -0
- package/dist/core/quick.d.ts +62 -0
- package/dist/core/quick.d.ts.map +1 -0
- package/dist/core/quick.js +300 -0
- package/dist/core/quick.js.map +1 -0
- package/dist/core/retrieval.d.ts +29 -0
- package/dist/core/retrieval.d.ts.map +1 -0
- package/dist/core/retrieval.js +150 -0
- package/dist/core/retrieval.js.map +1 -0
- package/dist/core/runtime.d.ts +67 -0
- package/dist/core/runtime.d.ts.map +1 -0
- package/dist/core/runtime.js +84 -0
- package/dist/core/runtime.js.map +1 -0
- package/dist/core/streaming.d.ts +37 -0
- package/dist/core/streaming.d.ts.map +1 -0
- package/dist/core/streaming.js +51 -0
- package/dist/core/streaming.js.map +1 -0
- package/dist/core/sync.d.ts +13 -0
- package/dist/core/sync.d.ts.map +1 -0
- package/dist/core/sync.js +46 -0
- package/dist/core/sync.js.map +1 -0
- package/dist/core/telemetry.d.ts +8 -0
- package/dist/core/telemetry.d.ts.map +1 -0
- package/dist/core/telemetry.js +14 -0
- package/dist/core/telemetry.js.map +1 -0
- package/dist/core/tokens.d.ts +8 -0
- package/dist/core/tokens.d.ts.map +1 -0
- package/dist/core/tokens.js +59 -0
- package/dist/core/tokens.js.map +1 -0
- package/dist/core/trust.d.ts +23 -0
- package/dist/core/trust.d.ts.map +1 -0
- package/dist/core/trust.js +164 -0
- package/dist/core/trust.js.map +1 -0
- package/dist/core/validation.d.ts +36 -0
- package/dist/core/validation.d.ts.map +1 -0
- package/dist/core/validation.js +185 -0
- package/dist/core/validation.js.map +1 -0
- package/dist/embeddings/local.d.ts +5 -0
- package/dist/embeddings/local.d.ts.map +1 -0
- package/dist/embeddings/local.js +128 -0
- package/dist/embeddings/local.js.map +1 -0
- package/dist/embeddings/openai.d.ts +26 -0
- package/dist/embeddings/openai.d.ts.map +1 -0
- package/dist/embeddings/openai.js +48 -0
- package/dist/embeddings/openai.js.map +1 -0
- package/dist/embeddings/resilience.d.ts +5 -0
- package/dist/embeddings/resilience.d.ts.map +1 -0
- package/dist/embeddings/resilience.js +53 -0
- package/dist/embeddings/resilience.js.map +1 -0
- package/dist/embeddings/voyage.d.ts +30 -0
- package/dist/embeddings/voyage.d.ts.map +1 -0
- package/dist/embeddings/voyage.js +53 -0
- package/dist/embeddings/voyage.js.map +1 -0
- package/dist/index.d.ts +72 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +40 -0
- package/dist/index.js.map +1 -0
- package/dist/integrations/claude-agent.d.ts +21 -0
- package/dist/integrations/claude-agent.d.ts.map +1 -0
- package/dist/integrations/claude-agent.js +44 -0
- package/dist/integrations/claude-agent.js.map +1 -0
- package/dist/integrations/claude-tools.d.ts +18 -0
- package/dist/integrations/claude-tools.d.ts.map +1 -0
- package/dist/integrations/claude-tools.js +60 -0
- package/dist/integrations/claude-tools.js.map +1 -0
- package/dist/integrations/langchain.d.ts +24 -0
- package/dist/integrations/langchain.d.ts.map +1 -0
- package/dist/integrations/langchain.js +48 -0
- package/dist/integrations/langchain.js.map +1 -0
- package/dist/integrations/mcp.d.ts +23 -0
- package/dist/integrations/mcp.d.ts.map +1 -0
- package/dist/integrations/mcp.js +60 -0
- package/dist/integrations/mcp.js.map +1 -0
- package/dist/integrations/middleware.d.ts +15 -0
- package/dist/integrations/middleware.d.ts.map +1 -0
- package/dist/integrations/middleware.js +27 -0
- package/dist/integrations/middleware.js.map +1 -0
- package/dist/integrations/openai-tools.d.ts +21 -0
- package/dist/integrations/openai-tools.d.ts.map +1 -0
- package/dist/integrations/openai-tools.js +69 -0
- package/dist/integrations/openai-tools.js.map +1 -0
- package/dist/integrations/vercel-ai.d.ts +19 -0
- package/dist/integrations/vercel-ai.d.ts.map +1 -0
- package/dist/integrations/vercel-ai.js +41 -0
- package/dist/integrations/vercel-ai.js.map +1 -0
- package/dist/server/http-server.d.ts +61 -0
- package/dist/server/http-server.d.ts.map +1 -0
- package/dist/server/http-server.js +684 -0
- package/dist/server/http-server.js.map +1 -0
- package/dist/server/index.d.ts +5 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +3 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/mcp-server.d.ts +61 -0
- package/dist/server/mcp-server.d.ts.map +1 -0
- package/dist/server/mcp-server.js +465 -0
- package/dist/server/mcp-server.js.map +1 -0
- package/dist/summarizers/claude.d.ts +11 -0
- package/dist/summarizers/claude.d.ts.map +1 -0
- package/dist/summarizers/claude.js +39 -0
- package/dist/summarizers/claude.js.map +1 -0
- package/dist/summarizers/client.d.ts +23 -0
- package/dist/summarizers/client.d.ts.map +1 -0
- package/dist/summarizers/client.js +24 -0
- package/dist/summarizers/client.js.map +1 -0
- package/dist/summarizers/extractive.d.ts +6 -0
- package/dist/summarizers/extractive.d.ts.map +1 -0
- package/dist/summarizers/extractive.js +204 -0
- package/dist/summarizers/extractive.js.map +1 -0
- package/dist/summarizers/extractor.d.ts +12 -0
- package/dist/summarizers/extractor.d.ts.map +1 -0
- package/dist/summarizers/extractor.js +75 -0
- package/dist/summarizers/extractor.js.map +1 -0
- package/dist/summarizers/openai.d.ts +11 -0
- package/dist/summarizers/openai.d.ts.map +1 -0
- package/dist/summarizers/openai.js +41 -0
- package/dist/summarizers/openai.js.map +1 -0
- package/dist/summarizers/prompts.d.ts +11 -0
- package/dist/summarizers/prompts.d.ts.map +1 -0
- package/dist/summarizers/prompts.js +104 -0
- package/dist/summarizers/prompts.js.map +1 -0
- package/docs/DEPLOYMENT.md +84 -0
- package/docs/INTEGRATIONS.md +64 -0
- package/docs/MEMORY_QUALITY_BASELINE.md +55 -0
- package/docs/MEMORY_QUALITY_RELEASE_GATE.md +63 -0
- package/docs/MEMORY_QUALITY_RUBRIC.md +249 -0
- package/docs/OPERATIONS.md +49 -0
- package/docs/SECURITY.md +25 -0
- package/openapi.yaml +843 -0
- package/package.json +157 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { normalizeScope } from '../contracts/identity.js';
|
|
2
|
+
import { COMPACTION_STATES, COMPACTION_TRIGGERS, FACT_CONFIDENCES, FACT_SOURCES, FACT_TYPES, EVIDENCE_SOURCE_TYPES, GROUNDING_STRENGTHS, KNOWLEDGE_AUDIT_DECISIONS, KNOWLEDGE_CLASSES, KNOWLEDGE_STATES, SUPPORT_POLARITIES, TURN_ROLES, WORK_ITEM_KINDS, WORK_ITEM_STATUSES, } from '../contracts/types.js';
|
|
3
|
+
export function nowSeconds() {
|
|
4
|
+
return Math.floor(Date.now() / 1000);
|
|
5
|
+
}
|
|
6
|
+
export function assertNonEmpty(value, name) {
|
|
7
|
+
if (!value || value.trim().length === 0) {
|
|
8
|
+
throw new Error(`Memory validation: '${name}' must not be empty`);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
export function assertEnum(value, allowed, name) {
|
|
12
|
+
if (!allowed.includes(value)) {
|
|
13
|
+
throw new Error(`Memory validation: '${name}' must be one of [${allowed.join(', ')}], got '${value}'`);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export function assertStringArray(value, name) {
|
|
17
|
+
if (!Array.isArray(value) || value.some((item) => typeof item !== 'string')) {
|
|
18
|
+
throw new Error(`Memory validation: '${name}' must be an array of strings`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function assertMaxEntries(value, name, max) {
|
|
22
|
+
if (value.length > max) {
|
|
23
|
+
throw new Error(`Memory validation: '${name}' must have at most ${max} entries, got ${value.length}`);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export function assertNumberRange(value, name, min, max) {
|
|
27
|
+
if (!Number.isFinite(value) || value < min || value > max) {
|
|
28
|
+
throw new Error(`Memory validation: '${name}' must be between ${min} and ${max}, got '${value}'`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export function assertTurnRange(startId, endId) {
|
|
32
|
+
if (endId < startId) {
|
|
33
|
+
throw new Error(`Memory validation: 'turn_id_end' (${endId}) must be >= 'turn_id_start' (${startId})`);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export function assertScope(scope) {
|
|
37
|
+
return normalizeScope(scope);
|
|
38
|
+
}
|
|
39
|
+
export function assertTurnRole(role) {
|
|
40
|
+
assertEnum(role, TURN_ROLES, 'role');
|
|
41
|
+
}
|
|
42
|
+
export function assertCompactionTrigger(trigger, name = 'compaction_trigger') {
|
|
43
|
+
assertEnum(trigger, COMPACTION_TRIGGERS, name);
|
|
44
|
+
}
|
|
45
|
+
export function assertFactType(factType) {
|
|
46
|
+
assertEnum(factType, FACT_TYPES, 'fact_type');
|
|
47
|
+
}
|
|
48
|
+
export function assertFactSource(source) {
|
|
49
|
+
assertEnum(source, FACT_SOURCES, 'source');
|
|
50
|
+
}
|
|
51
|
+
export function assertFactConfidence(confidence) {
|
|
52
|
+
assertEnum(confidence, FACT_CONFIDENCES, 'confidence');
|
|
53
|
+
}
|
|
54
|
+
export function assertKnowledgeState(state) {
|
|
55
|
+
assertEnum(state, KNOWLEDGE_STATES, 'knowledge_state');
|
|
56
|
+
}
|
|
57
|
+
export function assertKnowledgeClass(knowledgeClass) {
|
|
58
|
+
assertEnum(knowledgeClass, KNOWLEDGE_CLASSES, 'knowledge_class');
|
|
59
|
+
}
|
|
60
|
+
export function assertEvidenceSourceType(sourceType) {
|
|
61
|
+
assertEnum(sourceType, EVIDENCE_SOURCE_TYPES, 'source_type');
|
|
62
|
+
}
|
|
63
|
+
export function assertGroundingStrength(strength) {
|
|
64
|
+
assertEnum(strength, GROUNDING_STRENGTHS, 'grounding_strength');
|
|
65
|
+
}
|
|
66
|
+
export function assertSupportPolarity(polarity) {
|
|
67
|
+
assertEnum(polarity, SUPPORT_POLARITIES, 'support_polarity');
|
|
68
|
+
}
|
|
69
|
+
export function assertKnowledgeAuditDecision(decision) {
|
|
70
|
+
assertEnum(decision, KNOWLEDGE_AUDIT_DECISIONS, 'decision');
|
|
71
|
+
}
|
|
72
|
+
export function assertCompactionState(state) {
|
|
73
|
+
assertEnum(state, COMPACTION_STATES, 'compaction_state');
|
|
74
|
+
}
|
|
75
|
+
export function assertWorkItemKind(kind) {
|
|
76
|
+
assertEnum(kind, WORK_ITEM_KINDS, 'kind');
|
|
77
|
+
}
|
|
78
|
+
export function assertWorkItemStatus(status) {
|
|
79
|
+
assertEnum(status, WORK_ITEM_STATUSES, 'status');
|
|
80
|
+
}
|
|
81
|
+
export function validateNewTurn(input) {
|
|
82
|
+
const scope = assertScope(input);
|
|
83
|
+
assertNonEmpty(input.session_id, 'session_id');
|
|
84
|
+
assertNonEmpty(input.actor, 'actor');
|
|
85
|
+
assertNonEmpty(input.content, 'content');
|
|
86
|
+
assertTurnRole(input.role);
|
|
87
|
+
if (input.priority !== undefined) {
|
|
88
|
+
assertNumberRange(input.priority, 'priority', 0, 2);
|
|
89
|
+
}
|
|
90
|
+
return scope;
|
|
91
|
+
}
|
|
92
|
+
export function validateNewWorkingMemory(input) {
|
|
93
|
+
const scope = assertScope(input);
|
|
94
|
+
assertNonEmpty(input.session_id, 'session_id');
|
|
95
|
+
assertNonEmpty(input.summary, 'summary');
|
|
96
|
+
assertCompactionTrigger(input.compaction_trigger);
|
|
97
|
+
assertStringArray(input.key_entities, 'key_entities');
|
|
98
|
+
assertStringArray(input.topic_tags, 'topic_tags');
|
|
99
|
+
assertMaxEntries(input.topic_tags, 'topic_tags', 5);
|
|
100
|
+
assertTurnRange(input.turn_id_start, input.turn_id_end);
|
|
101
|
+
return scope;
|
|
102
|
+
}
|
|
103
|
+
export function validateNewKnowledgeMemory(input) {
|
|
104
|
+
const scope = assertScope(input);
|
|
105
|
+
assertNonEmpty(input.fact, 'fact');
|
|
106
|
+
assertFactType(input.fact_type);
|
|
107
|
+
assertFactSource(input.source);
|
|
108
|
+
assertFactConfidence(input.confidence);
|
|
109
|
+
if (input.knowledge_state !== undefined) {
|
|
110
|
+
assertKnowledgeState(input.knowledge_state);
|
|
111
|
+
}
|
|
112
|
+
if (input.knowledge_class !== undefined) {
|
|
113
|
+
assertKnowledgeClass(input.knowledge_class);
|
|
114
|
+
}
|
|
115
|
+
if (input.grounding_strength !== undefined) {
|
|
116
|
+
assertGroundingStrength(input.grounding_strength);
|
|
117
|
+
}
|
|
118
|
+
return scope;
|
|
119
|
+
}
|
|
120
|
+
export function validateNewKnowledgeMemoryAudit(input) {
|
|
121
|
+
const scope = assertScope(input);
|
|
122
|
+
assertNonEmpty(input.fact, 'fact');
|
|
123
|
+
assertFactType(input.fact_type);
|
|
124
|
+
assertFactConfidence(input.confidence);
|
|
125
|
+
assertKnowledgeAuditDecision(input.decision);
|
|
126
|
+
return scope;
|
|
127
|
+
}
|
|
128
|
+
export function validateNewKnowledgeCandidate(input) {
|
|
129
|
+
const scope = assertScope(input);
|
|
130
|
+
assertNonEmpty(input.fact, 'fact');
|
|
131
|
+
assertFactType(input.fact_type);
|
|
132
|
+
assertKnowledgeClass(input.knowledge_class);
|
|
133
|
+
assertFactConfidence(input.confidence);
|
|
134
|
+
assertGroundingStrength(input.grounding_strength ?? 'weak');
|
|
135
|
+
assertKnowledgeState(input.state ?? 'candidate');
|
|
136
|
+
return scope;
|
|
137
|
+
}
|
|
138
|
+
export function validateNewKnowledgeEvidence(input) {
|
|
139
|
+
const scope = assertScope(input);
|
|
140
|
+
assertEvidenceSourceType(input.source_type);
|
|
141
|
+
assertSupportPolarity(input.support_polarity);
|
|
142
|
+
assertNonEmpty(input.excerpt, 'excerpt');
|
|
143
|
+
if (input.speaker_role !== undefined && input.speaker_role !== null) {
|
|
144
|
+
assertTurnRole(input.speaker_role);
|
|
145
|
+
}
|
|
146
|
+
return scope;
|
|
147
|
+
}
|
|
148
|
+
export function validateContextMonitorUpsert(input) {
|
|
149
|
+
const scope = assertScope(input);
|
|
150
|
+
assertCompactionState(input.compaction_state);
|
|
151
|
+
return scope;
|
|
152
|
+
}
|
|
153
|
+
export function validateNewCompactionLog(input) {
|
|
154
|
+
const scope = assertScope(input);
|
|
155
|
+
assertNonEmpty(input.session_id, 'session_id');
|
|
156
|
+
assertCompactionTrigger(input.trigger_type, 'trigger_type');
|
|
157
|
+
assertTurnRange(input.turn_id_start, input.turn_id_end);
|
|
158
|
+
return scope;
|
|
159
|
+
}
|
|
160
|
+
export function validateTimeRange(range) {
|
|
161
|
+
if (range.start_at !== undefined &&
|
|
162
|
+
range.end_at !== undefined &&
|
|
163
|
+
range.end_at < range.start_at) {
|
|
164
|
+
throw new Error("Memory validation: 'end_at' must be >= 'start_at'");
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
export function validateNewWorkItem(input) {
|
|
168
|
+
const scope = assertScope(input);
|
|
169
|
+
assertNonEmpty(input.title, 'title');
|
|
170
|
+
assertWorkItemKind(input.kind);
|
|
171
|
+
assertWorkItemStatus(input.status ?? 'open');
|
|
172
|
+
return scope;
|
|
173
|
+
}
|
|
174
|
+
export function assertArchiveInput(id, archivedAt, compactionLogId) {
|
|
175
|
+
if (!Number.isInteger(id) || id <= 0) {
|
|
176
|
+
throw new Error(`Memory validation: 'id' must be a positive integer, got '${id}'`);
|
|
177
|
+
}
|
|
178
|
+
if (!Number.isInteger(archivedAt) || archivedAt <= 0) {
|
|
179
|
+
throw new Error(`Memory validation: 'archivedAt' must be a positive integer, got '${archivedAt}'`);
|
|
180
|
+
}
|
|
181
|
+
if (!Number.isInteger(compactionLogId) || compactionLogId <= 0) {
|
|
182
|
+
throw new Error(`Memory validation: 'compactionLogId' must be a positive integer, got '${compactionLogId}'`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
//# sourceMappingURL=validation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validation.js","sourceRoot":"","sources":["../../src/core/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAgD,MAAM,0BAA0B,CAAC;AA2BxG,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,qBAAqB,EACrB,mBAAmB,EACnB,yBAAyB,EACzB,iBAAiB,EACjB,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,eAAe,EACf,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,UAAU,UAAU;IACxB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,IAAY;IACxD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,qBAAqB,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAI,KAAQ,EAAE,OAAqB,EAAE,IAAY;IACzE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,KAAK,GAAG,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAe,EAAE,IAAY;IAC7D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,+BAA+B,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAe,EAAE,IAAY,EAAE,GAAW;IACzE,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,uBAAuB,GAAG,iBAAiB,KAAK,CAAC,MAAM,EAAE,CACrF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,IAAY,EAAE,GAAW,EAAE,GAAW;IACrF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,qBAAqB,GAAG,QAAQ,GAAG,UAAU,KAAK,GAAG,CACjF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAe,EAAE,KAAa;IAC5D,IAAI,KAAK,GAAG,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACb,qCAAqC,KAAK,iCAAiC,OAAO,GAAG,CACtF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAkB;IAC5C,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAc;IAC3C,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,OAA0B,EAAE,IAAI,GAAG,oBAAoB;IAC7F,UAAU,CAAC,OAAO,EAAE,mBAAmB,EAAE,IAAI,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAAkB;IAC/C,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAA0B;IAC7D,UAAU,CAAC,UAAU,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,KAAqB;IACxD,UAAU,CAAC,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,cAA8B;IACjE,UAAU,CAAC,cAAc,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,UAA8B;IACrE,UAAU,CAAC,UAAU,EAAE,qBAAqB,EAAE,aAAa,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,QAA2B;IACjE,UAAU,CAAC,QAAQ,EAAE,mBAAmB,EAAE,oBAAoB,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAoC;IACxE,UAAU,CAAC,QAAQ,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,QAAgC;IAC3E,UAAU,CAAC,QAAQ,EAAE,yBAAyB,EAAE,UAAU,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAsB;IAC1D,UAAU,CAAC,KAAK,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAkB;IACnD,UAAU,CAAC,IAAI,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAsB;IACzD,UAAU,CAAC,MAAM,EAAE,kBAAkB,EAAE,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/C,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACrC,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,KAAK,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjC,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,KAAuB;IAC9D,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/C,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzC,uBAAuB,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAClD,iBAAiB,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;IACtD,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAClD,gBAAgB,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;IACpD,eAAe,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IACxD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,KAAyB;IAClE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAChC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,oBAAoB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QACxC,oBAAoB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QACxC,oBAAoB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC3C,uBAAuB,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,+BAA+B,CAC7C,KAA8B;IAE9B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAChC,oBAAoB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,4BAA4B,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,KAA4B;IACxE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACnC,cAAc,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAChC,oBAAoB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAC5C,oBAAoB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,uBAAuB,CAAC,KAAK,CAAC,kBAAkB,IAAI,MAAM,CAAC,CAAC;IAC5D,oBAAoB,CAAC,KAAK,CAAC,KAAK,IAAI,WAAW,CAAC,CAAC;IACjD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAA2B;IACtE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,wBAAwB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5C,qBAAqB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC9C,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,IAAI,KAAK,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QACpE,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAA2B;IACtE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,qBAAqB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IAC9C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,KAAuB;IAC9D,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/C,uBAAuB,CAAC,KAAK,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;IAC5D,eAAe,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IACxD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAgB;IAChD,IACE,KAAK,CAAC,QAAQ,KAAK,SAAS;QAC5B,KAAK,CAAC,MAAM,KAAK,SAAS;QAC1B,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,EAC7B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAkB;IACpD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;IACjC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACrC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,oBAAoB,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC;IAC7C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,EAAU,EAAE,UAAkB,EAAE,eAAuB;IACxF,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,4DAA4D,EAAE,GAAG,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CACb,oEAAoE,UAAU,GAAG,CAClF,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CACb,yEAAyE,eAAe,GAAG,CAC5F,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/embeddings/local.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAmB,MAAM,2BAA2B,CAAC;AAsHrF,wBAAgB,6BAA6B,CAAC,OAAO,CAAC,EAAE;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,kBAAkB,CA4BrB"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
const DEFAULT_DIMENSIONS = 384;
|
|
2
|
+
const HASH_SEEDS = [2166136261, 16777619, 374761393, 668265263];
|
|
3
|
+
const TOKEN_ALIASES = {
|
|
4
|
+
pg: ['postgres', 'postgresql'],
|
|
5
|
+
postgres: ['postgresql', 'pg'],
|
|
6
|
+
postgresql: ['postgres', 'pg'],
|
|
7
|
+
ts: ['typescript'],
|
|
8
|
+
typescript: ['ts'],
|
|
9
|
+
js: ['javascript'],
|
|
10
|
+
javascript: ['js'],
|
|
11
|
+
auth: ['authentication'],
|
|
12
|
+
authentication: ['auth'],
|
|
13
|
+
repo: ['repository'],
|
|
14
|
+
repository: ['repo'],
|
|
15
|
+
deploy: ['deployment', 'release'],
|
|
16
|
+
deployment: ['deploy', 'release'],
|
|
17
|
+
bug: ['issue'],
|
|
18
|
+
issue: ['bug'],
|
|
19
|
+
};
|
|
20
|
+
function tokenize(text) {
|
|
21
|
+
return text
|
|
22
|
+
.toLowerCase()
|
|
23
|
+
.split(/[^a-z0-9]+/g)
|
|
24
|
+
.filter((token) => token.length > 1);
|
|
25
|
+
}
|
|
26
|
+
function stemToken(token) {
|
|
27
|
+
return token
|
|
28
|
+
.replace(/(ments|ment|tions|tion|ings|ing|edly|edly|edly|edly|ed|ly|es|s)$/g, '')
|
|
29
|
+
.replace(/(.)\1$/g, '$1');
|
|
30
|
+
}
|
|
31
|
+
function expandTokenFeatures(token) {
|
|
32
|
+
const features = new Set([token]);
|
|
33
|
+
const stem = stemToken(token);
|
|
34
|
+
if (stem.length > 2) {
|
|
35
|
+
features.add(`stem:${stem}`);
|
|
36
|
+
}
|
|
37
|
+
for (const alias of TOKEN_ALIASES[token] ?? []) {
|
|
38
|
+
features.add(`alias:${alias}`);
|
|
39
|
+
const aliasStem = stemToken(alias);
|
|
40
|
+
if (aliasStem.length > 2) {
|
|
41
|
+
features.add(`stem:${aliasStem}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return [...features];
|
|
45
|
+
}
|
|
46
|
+
function charTrigrams(token) {
|
|
47
|
+
if (token.length <= 3)
|
|
48
|
+
return [token];
|
|
49
|
+
const padded = `_${token}_`;
|
|
50
|
+
const results = [];
|
|
51
|
+
for (let index = 0; index <= padded.length - 3; index += 1) {
|
|
52
|
+
results.push(padded.slice(index, index + 3));
|
|
53
|
+
}
|
|
54
|
+
return results;
|
|
55
|
+
}
|
|
56
|
+
function collectFeatures(tokens) {
|
|
57
|
+
const features = new Map();
|
|
58
|
+
const increment = (feature) => {
|
|
59
|
+
features.set(feature, (features.get(feature) ?? 0) + 1);
|
|
60
|
+
};
|
|
61
|
+
tokens.forEach((token, index) => {
|
|
62
|
+
const expanded = expandTokenFeatures(token);
|
|
63
|
+
expanded.forEach((feature) => increment(`tok:${feature}`));
|
|
64
|
+
for (const feature of expanded) {
|
|
65
|
+
for (const trigram of charTrigrams(feature.replace(/^(?:stem:|alias:)/, ''))) {
|
|
66
|
+
increment(`chr:${trigram}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (index < tokens.length - 1) {
|
|
70
|
+
increment(`big:${token}_${tokens[index + 1]}`);
|
|
71
|
+
}
|
|
72
|
+
if (index < tokens.length - 2) {
|
|
73
|
+
increment(`skip:${token}_${tokens[index + 2]}`);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
return features;
|
|
77
|
+
}
|
|
78
|
+
function hashFeature(feature, dimensions, seed) {
|
|
79
|
+
let hash = seed;
|
|
80
|
+
for (let i = 0; i < feature.length; i += 1) {
|
|
81
|
+
hash ^= feature.charCodeAt(i);
|
|
82
|
+
hash = Math.imul(hash, 16777619);
|
|
83
|
+
}
|
|
84
|
+
const normalized = hash >>> 0;
|
|
85
|
+
return {
|
|
86
|
+
index: normalized % dimensions,
|
|
87
|
+
sign: normalized % 2 === 0 ? 1 : -1,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
function normalizeVector(vector) {
|
|
91
|
+
let magnitude = 0;
|
|
92
|
+
for (let i = 0; i < vector.length; i += 1) {
|
|
93
|
+
magnitude += vector[i] * vector[i];
|
|
94
|
+
}
|
|
95
|
+
if (magnitude === 0) {
|
|
96
|
+
return vector;
|
|
97
|
+
}
|
|
98
|
+
const scale = Math.sqrt(magnitude);
|
|
99
|
+
for (let i = 0; i < vector.length; i += 1) {
|
|
100
|
+
vector[i] /= scale;
|
|
101
|
+
}
|
|
102
|
+
return vector;
|
|
103
|
+
}
|
|
104
|
+
export function createLocalEmbeddingGenerator(options) {
|
|
105
|
+
const dimensions = options?.dimensions ?? DEFAULT_DIMENSIONS;
|
|
106
|
+
return async (texts) => {
|
|
107
|
+
const featureSets = texts.map((text) => collectFeatures(tokenize(text)));
|
|
108
|
+
const documentFrequency = new Map();
|
|
109
|
+
featureSets.forEach((features) => {
|
|
110
|
+
features.forEach((_count, feature) => {
|
|
111
|
+
documentFrequency.set(feature, (documentFrequency.get(feature) ?? 0) + 1);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
return featureSets.map((features) => {
|
|
115
|
+
const vector = new Float32Array(dimensions);
|
|
116
|
+
for (const [feature, termFrequency] of features.entries()) {
|
|
117
|
+
const inverseDocumentFrequency = 1 + Math.log((1 + featureSets.length) / (1 + (documentFrequency.get(feature) ?? 0)));
|
|
118
|
+
const weight = (1 + Math.log(termFrequency)) * inverseDocumentFrequency;
|
|
119
|
+
for (const seed of HASH_SEEDS) {
|
|
120
|
+
const { index, sign } = hashFeature(feature, dimensions, seed);
|
|
121
|
+
vector[index] += weight * sign;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return normalizeVector(vector);
|
|
125
|
+
});
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=local.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local.js","sourceRoot":"","sources":["../../src/embeddings/local.ts"],"names":[],"mappings":"AAEA,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,UAAU,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;AAChE,MAAM,aAAa,GAA6B;IAC9C,EAAE,EAAE,CAAC,UAAU,EAAE,YAAY,CAAC;IAC9B,QAAQ,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC;IAC9B,UAAU,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC;IAC9B,EAAE,EAAE,CAAC,YAAY,CAAC;IAClB,UAAU,EAAE,CAAC,IAAI,CAAC;IAClB,EAAE,EAAE,CAAC,YAAY,CAAC;IAClB,UAAU,EAAE,CAAC,IAAI,CAAC;IAClB,IAAI,EAAE,CAAC,gBAAgB,CAAC;IACxB,cAAc,EAAE,CAAC,MAAM,CAAC;IACxB,IAAI,EAAE,CAAC,YAAY,CAAC;IACpB,UAAU,EAAE,CAAC,MAAM,CAAC;IACpB,MAAM,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;IACjC,UAAU,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;IACjC,GAAG,EAAE,CAAC,OAAO,CAAC;IACd,KAAK,EAAE,CAAC,KAAK,CAAC;CACf,CAAC;AAEF,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,aAAa,CAAC;SACpB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK;SACT,OAAO,CAAC,mEAAmE,EAAE,EAAE,CAAC;SAChF,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,QAAQ,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/C,QAAQ,CAAC,GAAG,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,GAAG,CAAC,QAAQ,SAAS,EAAE,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,KAAK,GAAG,CAAC;IAC5B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,eAAe,CAAC,MAAgB;IACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,MAAM,SAAS,GAAG,CAAC,OAAe,EAAE,EAAE;QACpC,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC;IAEF,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5C,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC,CAAC;QAC3D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,SAAS,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,OAAO,KAAK,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,QAAQ,KAAK,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,OAAe,EAAE,UAAkB,EAAE,IAAY;IAIpE,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,CAAC;IAC9B,OAAO;QACL,KAAK,EAAE,UAAU,GAAG,UAAU;QAC9B,IAAI,EAAE,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,MAAoB;IAC3C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,OAE7C;IACC,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,kBAAkB,CAAC;IAE7D,OAAO,KAAK,EAAE,KAAe,EAA8B,EAAE;QAC3D,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;QACpD,WAAW,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC/B,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;gBACnC,iBAAiB,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC5E,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAClC,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;YAE5C,KAAK,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC1D,MAAM,wBAAwB,GAC5B,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvF,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,wBAAwB,CAAC;gBACxE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oBAC9B,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;oBAC/D,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,GAAG,IAAI,CAAC;gBACjC,CAAC;YACH,CAAC;YAED,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { EmbeddingGenerator } from '../contracts/embedding.js';
|
|
2
|
+
export interface OpenAIEmbeddingOptions {
|
|
3
|
+
/** API key. Defaults to OPENAI_API_KEY env var. */
|
|
4
|
+
apiKey?: string;
|
|
5
|
+
/** Model to use. Defaults to 'text-embedding-3-small'. */
|
|
6
|
+
model?: string;
|
|
7
|
+
/** Optional dimensions override (for models that support it). */
|
|
8
|
+
dimensions?: number;
|
|
9
|
+
maxRetries?: number;
|
|
10
|
+
retryDelayMs?: number;
|
|
11
|
+
batchSize?: number;
|
|
12
|
+
cacheSize?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Creates an EmbeddingGenerator that calls the OpenAI embeddings API.
|
|
16
|
+
* Requires the `openai` package as an optional peer dependency.
|
|
17
|
+
*
|
|
18
|
+
* ```typescript
|
|
19
|
+
* import { createOpenAIEmbeddingGenerator } from 'memory-layer/embeddings/openai';
|
|
20
|
+
*
|
|
21
|
+
* const generator = createOpenAIEmbeddingGenerator();
|
|
22
|
+
* const vectors = await generator(['hello world']);
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export declare function createOpenAIEmbeddingGenerator(options?: OpenAIEmbeddingOptions): EmbeddingGenerator;
|
|
26
|
+
//# sourceMappingURL=openai.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/embeddings/openai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAmB,MAAM,2BAA2B,CAAC;AAOrF,MAAM,WAAW,sBAAsB;IACrC,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,CAAC,EAAE,sBAAsB,GAC/B,kBAAkB,CA8DpB"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { batchedGenerate, createCachedEmbeddingGenerator, withRetry, } from './resilience.js';
|
|
2
|
+
/**
|
|
3
|
+
* Creates an EmbeddingGenerator that calls the OpenAI embeddings API.
|
|
4
|
+
* Requires the `openai` package as an optional peer dependency.
|
|
5
|
+
*
|
|
6
|
+
* ```typescript
|
|
7
|
+
* import { createOpenAIEmbeddingGenerator } from 'memory-layer/embeddings/openai';
|
|
8
|
+
*
|
|
9
|
+
* const generator = createOpenAIEmbeddingGenerator();
|
|
10
|
+
* const vectors = await generator(['hello world']);
|
|
11
|
+
* ```
|
|
12
|
+
*/
|
|
13
|
+
export function createOpenAIEmbeddingGenerator(options) {
|
|
14
|
+
const model = options?.model ?? 'text-embedding-3-small';
|
|
15
|
+
const dimensions = options?.dimensions;
|
|
16
|
+
let clientPromise = null;
|
|
17
|
+
async function getClient() {
|
|
18
|
+
if (!clientPromise) {
|
|
19
|
+
clientPromise = (async () => {
|
|
20
|
+
try {
|
|
21
|
+
const moduleName = 'openai';
|
|
22
|
+
const mod = await import(moduleName);
|
|
23
|
+
const OpenAI = mod.default ?? mod;
|
|
24
|
+
return new OpenAI({ apiKey: options?.apiKey });
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
throw new Error('memory-layer: OpenAI embedding generator requires the "openai" package. ' +
|
|
28
|
+
'Install it with: npm install openai');
|
|
29
|
+
}
|
|
30
|
+
})();
|
|
31
|
+
}
|
|
32
|
+
return clientPromise;
|
|
33
|
+
}
|
|
34
|
+
const baseGenerator = async (texts) => {
|
|
35
|
+
if (texts.length === 0)
|
|
36
|
+
return [];
|
|
37
|
+
const client = await getClient();
|
|
38
|
+
const response = await client.embeddings.create({
|
|
39
|
+
model,
|
|
40
|
+
input: texts,
|
|
41
|
+
...(dimensions ? { dimensions } : {}),
|
|
42
|
+
});
|
|
43
|
+
return response.data.map((item) => new Float32Array(item.embedding));
|
|
44
|
+
};
|
|
45
|
+
const cachedGenerator = createCachedEmbeddingGenerator(baseGenerator, options?.cacheSize ?? 256);
|
|
46
|
+
return async (texts) => withRetry(() => batchedGenerate(cachedGenerator, texts, options?.batchSize ?? 32), options?.maxRetries ?? 2, options?.retryDelayMs ?? 250);
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=openai.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/embeddings/openai.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,8BAA8B,EAC9B,SAAS,GACV,MAAM,iBAAiB,CAAC;AAezB;;;;;;;;;;GAUG;AACH,MAAM,UAAU,8BAA8B,CAC5C,OAAgC;IAEhC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,wBAAwB,CAAC;IACzD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,CAAC;IAEvC,IAAI,aAAa,GAA4B,IAAI,CAAC;IAElD,KAAK,UAAU,SAAS;QACtB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,aAAa,GAAG,CAAC,KAAK,IAAI,EAAE;gBAC1B,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,QAAQ,CAAC;oBAC5B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;oBACrC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC;oBAClC,OAAO,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,IAAI,KAAK,CACb,0EAA0E;wBACxE,qCAAqC,CACxC,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,MAAM,aAAa,GAAuB,KAAK,EAAE,KAAe,EAA8B,EAAE;QAC9F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,MAAM,GAAG,MAAM,SAAS,EAU7B,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YAC9C,KAAK;YACL,KAAK,EAAE,KAAK;YACZ,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CACtB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAC3C,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,8BAA8B,CACpD,aAAa,EACb,OAAO,EAAE,SAAS,IAAI,GAAG,CAC1B,CAAC;IAEF,OAAO,KAAK,EAAE,KAAe,EAA8B,EAAE,CAC3D,SAAS,CACP,GAAG,EAAE,CAAC,eAAe,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC,EACvE,OAAO,EAAE,UAAU,IAAI,CAAC,EACxB,OAAO,EAAE,YAAY,IAAI,GAAG,CAC7B,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { EmbeddingGenerator, EmbeddingVector } from '../contracts/embedding.js';
|
|
2
|
+
export declare function withRetry<T>(fn: () => Promise<T>, maxRetries?: number, delayMs?: number): Promise<T>;
|
|
3
|
+
export declare function batchedGenerate(generator: EmbeddingGenerator, inputs: string[], batchSize?: number): Promise<EmbeddingVector[]>;
|
|
4
|
+
export declare function createCachedEmbeddingGenerator(generator: EmbeddingGenerator, maxEntries?: number): EmbeddingGenerator;
|
|
5
|
+
//# sourceMappingURL=resilience.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resilience.d.ts","sourceRoot":"","sources":["../../src/embeddings/resilience.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAMrF,wBAAsB,SAAS,CAAC,CAAC,EAC/B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,UAAU,SAAI,EACd,OAAO,SAAM,GACZ,OAAO,CAAC,CAAC,CAAC,CAcZ;AAED,wBAAsB,eAAe,CACnC,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,SAAS,SAAK,GACb,OAAO,CAAC,eAAe,EAAE,CAAC,CAU5B;AAED,wBAAgB,8BAA8B,CAC5C,SAAS,EAAE,kBAAkB,EAC7B,UAAU,SAAM,GACf,kBAAkB,CAoBpB"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
function sleep(ms) {
|
|
2
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3
|
+
}
|
|
4
|
+
export async function withRetry(fn, maxRetries = 2, delayMs = 250) {
|
|
5
|
+
let lastError;
|
|
6
|
+
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
|
|
7
|
+
try {
|
|
8
|
+
return await fn();
|
|
9
|
+
}
|
|
10
|
+
catch (error) {
|
|
11
|
+
lastError = error;
|
|
12
|
+
if (attempt === maxRetries) {
|
|
13
|
+
throw error;
|
|
14
|
+
}
|
|
15
|
+
await sleep(delayMs * 2 ** attempt);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
throw lastError;
|
|
19
|
+
}
|
|
20
|
+
export async function batchedGenerate(generator, inputs, batchSize = 32) {
|
|
21
|
+
if (inputs.length === 0)
|
|
22
|
+
return [];
|
|
23
|
+
if (batchSize <= 0)
|
|
24
|
+
return generator(inputs);
|
|
25
|
+
const output = [];
|
|
26
|
+
for (let index = 0; index < inputs.length; index += batchSize) {
|
|
27
|
+
const batch = inputs.slice(index, index + batchSize);
|
|
28
|
+
output.push(...(await generator(batch)));
|
|
29
|
+
}
|
|
30
|
+
return output;
|
|
31
|
+
}
|
|
32
|
+
export function createCachedEmbeddingGenerator(generator, maxEntries = 256) {
|
|
33
|
+
const cache = new Map();
|
|
34
|
+
return async (texts) => {
|
|
35
|
+
const missing = texts.filter((text) => !cache.has(text));
|
|
36
|
+
if (missing.length > 0) {
|
|
37
|
+
const generated = await generator(missing);
|
|
38
|
+
missing.forEach((text, index) => {
|
|
39
|
+
const vector = generated[index];
|
|
40
|
+
if (!vector)
|
|
41
|
+
return;
|
|
42
|
+
cache.set(text, vector);
|
|
43
|
+
if (cache.size > maxEntries) {
|
|
44
|
+
const firstKey = cache.keys().next().value;
|
|
45
|
+
if (firstKey)
|
|
46
|
+
cache.delete(firstKey);
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
return texts.map((text) => cache.get(text) ?? new Float32Array());
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=resilience.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resilience.js","sourceRoot":"","sources":["../../src/embeddings/resilience.ts"],"names":[],"mappings":"AAEA,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAoB,EACpB,UAAU,GAAG,CAAC,EACd,OAAO,GAAG,GAAG;IAEb,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;gBAC3B,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,KAAK,CAAC,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IACD,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAA6B,EAC7B,MAAgB,EAChB,SAAS,GAAG,EAAE;IAEd,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;QACrD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,SAA6B,EAC7B,UAAU,GAAG,GAAG;IAEhB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IAEjD,OAAO,KAAK,EAAE,KAAe,EAA8B,EAAE;QAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;YAC3C,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,MAAM;oBAAE,OAAO;gBACpB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACxB,IAAI,KAAK,CAAC,IAAI,GAAG,UAAU,EAAE,CAAC;oBAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;oBAC3C,IAAI,QAAQ;wBAAE,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,YAAY,EAAE,CAAC,CAAC;IACpE,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { EmbeddingGenerator } from '../contracts/embedding.js';
|
|
2
|
+
export interface VoyageEmbeddingOptions {
|
|
3
|
+
/** Voyage API key. Defaults to VOYAGE_API_KEY env var. */
|
|
4
|
+
apiKey?: string;
|
|
5
|
+
/** Model to use. Defaults to 'voyage-3-lite'. */
|
|
6
|
+
model?: string;
|
|
7
|
+
/** API base URL. Defaults to 'https://api.voyageai.com/v1'. */
|
|
8
|
+
baseUrl?: string;
|
|
9
|
+
maxRetries?: number;
|
|
10
|
+
retryDelayMs?: number;
|
|
11
|
+
batchSize?: number;
|
|
12
|
+
cacheSize?: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Creates an EmbeddingGenerator that calls the Voyage AI embeddings API.
|
|
16
|
+
* Voyage is Anthropic's recommended embedding provider.
|
|
17
|
+
*
|
|
18
|
+
* This generator uses raw HTTP (no SDK dependency) to minimize installation footprint.
|
|
19
|
+
*
|
|
20
|
+
* ```typescript
|
|
21
|
+
* import { createVoyageEmbeddingGenerator } from 'memory-layer/embeddings/voyage';
|
|
22
|
+
*
|
|
23
|
+
* const generator = createVoyageEmbeddingGenerator({
|
|
24
|
+
* apiKey: process.env.VOYAGE_API_KEY,
|
|
25
|
+
* });
|
|
26
|
+
* const vectors = await generator(['hello world']);
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
export declare function createVoyageEmbeddingGenerator(options?: VoyageEmbeddingOptions): EmbeddingGenerator;
|
|
30
|
+
//# sourceMappingURL=voyage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voyage.d.ts","sourceRoot":"","sources":["../../src/embeddings/voyage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAmB,MAAM,2BAA2B,CAAC;AAOrF,MAAM,WAAW,sBAAsB;IACrC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iDAAiD;IACjD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,CAAC,EAAE,sBAAsB,GAC/B,kBAAkB,CA0DpB"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { batchedGenerate, createCachedEmbeddingGenerator, withRetry, } from './resilience.js';
|
|
2
|
+
/**
|
|
3
|
+
* Creates an EmbeddingGenerator that calls the Voyage AI embeddings API.
|
|
4
|
+
* Voyage is Anthropic's recommended embedding provider.
|
|
5
|
+
*
|
|
6
|
+
* This generator uses raw HTTP (no SDK dependency) to minimize installation footprint.
|
|
7
|
+
*
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import { createVoyageEmbeddingGenerator } from 'memory-layer/embeddings/voyage';
|
|
10
|
+
*
|
|
11
|
+
* const generator = createVoyageEmbeddingGenerator({
|
|
12
|
+
* apiKey: process.env.VOYAGE_API_KEY,
|
|
13
|
+
* });
|
|
14
|
+
* const vectors = await generator(['hello world']);
|
|
15
|
+
* ```
|
|
16
|
+
*/
|
|
17
|
+
export function createVoyageEmbeddingGenerator(options) {
|
|
18
|
+
const model = options?.model ?? 'voyage-3-lite';
|
|
19
|
+
const baseUrl = options?.baseUrl ?? 'https://api.voyageai.com/v1';
|
|
20
|
+
function getApiKey() {
|
|
21
|
+
const key = options?.apiKey ?? process.env.VOYAGE_API_KEY;
|
|
22
|
+
if (!key) {
|
|
23
|
+
throw new Error('memory-layer: Voyage embedding generator requires an API key. ' +
|
|
24
|
+
'Set VOYAGE_API_KEY env var or pass apiKey option.');
|
|
25
|
+
}
|
|
26
|
+
return key;
|
|
27
|
+
}
|
|
28
|
+
const baseGenerator = async (texts) => {
|
|
29
|
+
if (texts.length === 0)
|
|
30
|
+
return [];
|
|
31
|
+
const apiKey = getApiKey();
|
|
32
|
+
const response = await fetch(`${baseUrl}/embeddings`, {
|
|
33
|
+
method: 'POST',
|
|
34
|
+
headers: {
|
|
35
|
+
'Content-Type': 'application/json',
|
|
36
|
+
Authorization: `Bearer ${apiKey}`,
|
|
37
|
+
},
|
|
38
|
+
body: JSON.stringify({
|
|
39
|
+
model,
|
|
40
|
+
input: texts,
|
|
41
|
+
}),
|
|
42
|
+
});
|
|
43
|
+
if (!response.ok) {
|
|
44
|
+
const body = await response.text().catch(() => '');
|
|
45
|
+
throw new Error(`memory-layer: Voyage API error ${response.status}: ${body}`);
|
|
46
|
+
}
|
|
47
|
+
const data = (await response.json());
|
|
48
|
+
return data.data.map((item) => new Float32Array(item.embedding));
|
|
49
|
+
};
|
|
50
|
+
const cachedGenerator = createCachedEmbeddingGenerator(baseGenerator, options?.cacheSize ?? 256);
|
|
51
|
+
return async (texts) => withRetry(() => batchedGenerate(cachedGenerator, texts, options?.batchSize ?? 32), options?.maxRetries ?? 2, options?.retryDelayMs ?? 250);
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=voyage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voyage.js","sourceRoot":"","sources":["../../src/embeddings/voyage.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,EACf,8BAA8B,EAC9B,SAAS,GACV,MAAM,iBAAiB,CAAC;AAezB;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,8BAA8B,CAC5C,OAAgC;IAEhC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,eAAe,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,6BAA6B,CAAC;IAElE,SAAS,SAAS;QAChB,MAAM,GAAG,GAAG,OAAO,EAAE,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC1D,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CACb,gEAAgE;gBAC9D,mDAAmD,CACtD,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,aAAa,GAAuB,KAAK,EAAE,KAAe,EAA8B,EAAE;QAC9F,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,aAAa,EAAE;YACpD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,MAAM,EAAE;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,KAAK,EAAE,KAAK;aACb,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACnD,MAAM,IAAI,KAAK,CACb,kCAAkC,QAAQ,CAAC,MAAM,KAAK,IAAI,EAAE,CAC7D,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAClB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAC3C,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,eAAe,GAAG,8BAA8B,CACpD,aAAa,EACb,OAAO,EAAE,SAAS,IAAI,GAAG,CAC1B,CAAC;IAEF,OAAO,KAAK,EAAE,KAAe,EAA8B,EAAE,CAC3D,SAAS,CACP,GAAG,EAAE,CAAC,eAAe,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE,CAAC,EACvE,OAAO,EAAE,UAAU,IAAI,CAAC,EACxB,OAAO,EAAE,YAAY,IAAI,GAAG,CAC7B,CAAC;AACN,CAAC"}
|