@futdevpro/fdp-agent-memory 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +345 -0
- package/build/package.json +96 -0
- package/build/src/_assets/mcp-client-config/README.md +29 -0
- package/build/src/_assets/mcp-client-config/claude_desktop_config.json +15 -0
- package/build/src/_assets/mcp-client-config/mcp.json +15 -0
- package/build/src/_collections/config-catalog.const.js +180 -0
- package/build/src/_collections/config-error-codes.const.js +30 -0
- package/build/src/_collections/config-presets.const.js +25 -0
- package/build/src/_collections/error-banners.const.js +100 -0
- package/build/src/_collections/error-codes.const.js +150 -0
- package/build/src/_collections/fam-db-models.const.js +37 -0
- package/build/src/_collections/fam-entry-bootstrap.util.js +80 -0
- package/build/src/_collections/fam-error-context.util.js +90 -0
- package/build/src/_collections/fam-error-factory.util.js +64 -0
- package/build/src/_enums/fam-config-level.type-enum.js +15 -0
- package/build/src/_enums/fam-table.type-enum.js +20 -0
- package/build/src/_integration-tests/_helpers/fam-integration-test-setup.util.js +105 -0
- package/build/src/_models/data-models/fam-codebase.data-model.js +51 -0
- package/build/src/_models/data-models/fam-coding-patterns.data-model.js +58 -0
- package/build/src/_models/data-models/fam-config.data-model.js +68 -0
- package/build/src/_models/data-models/fam-documents.data-model.js +53 -0
- package/build/src/_models/data-models/fam-entry-base-properties.const.js +43 -0
- package/build/src/_models/data-models/fam-entry.data-model.js +81 -0
- package/build/src/_models/data-models/fam-error.data-model.js +88 -0
- package/build/src/_models/data-models/fam-ingest-run.data-model.js +74 -0
- package/build/src/_models/data-models/fam-knowledge.data-model.js +48 -0
- package/build/src/_models/data-models/fam-memory.data-model.js +55 -0
- package/build/src/_models/data-models/fam-reference.data-model.js +67 -0
- package/build/src/_models/data-models/fam-rules.data-model.js +51 -0
- package/build/src/_models/data-models/fam-scope.data-model.js +52 -0
- package/build/src/_models/interfaces/fam-common.interface.js +23 -0
- package/build/src/_models/interfaces/fam-config.interface.js +2 -0
- package/build/src/_models/interfaces/fam-error.interface.js +2 -0
- package/build/src/_modules/embedding/_collections/fam-embedding-pricing.const.js +22 -0
- package/build/src/_modules/embedding/_collections/fam-store-registry.const.js +63 -0
- package/build/src/_modules/embedding/_models/interfaces/fam-embedding-cost.interface.js +10 -0
- package/build/src/_modules/embedding/_models/interfaces/fam-embedding-provider.interface.js +2 -0
- package/build/src/_modules/embedding/_models/interfaces/fam-resolved-provider.interface.js +2 -0
- package/build/src/_modules/embedding/_services/fam-embedding-bootstrap.control-service.js +52 -0
- package/build/src/_modules/embedding/_services/fam-embedding-cost.control-service.js +175 -0
- package/build/src/_modules/embedding/_services/fam-embedding-pipeline.control-service.js +202 -0
- package/build/src/_modules/embedding/_services/fam-embedding-preset.control-service.js +66 -0
- package/build/src/_modules/embedding/_services/fam-embedding.control-service.js +253 -0
- package/build/src/_modules/embedding/_services/fam-entry.data-service.js +64 -0
- package/build/src/_modules/embedding/_services/fam-lmstudio-embedding.provider.js +112 -0
- package/build/src/_modules/embedding/_services/fam-mock-embedding.provider.js +64 -0
- package/build/src/_modules/embedding/_services/fam-openai-embedding.provider.js +64 -0
- package/build/src/_modules/embedding/_services/fam-vector-search.control-service.js +244 -0
- package/build/src/_modules/embedding/index.js +40 -0
- package/build/src/_modules/ingest/_collections/fam-content-hash.util.js +35 -0
- package/build/src/_modules/ingest/_collections/fam-file-routing.util.js +95 -0
- package/build/src/_modules/ingest/_collections/fam-glob-match.util.js +84 -0
- package/build/src/_modules/ingest/_collections/fam-md-chunker.util.js +164 -0
- package/build/src/_modules/ingest/_collections/fam-scan-path.util.js +91 -0
- package/build/src/_modules/ingest/_collections/fam-secret-exclude.util.js +54 -0
- package/build/src/_modules/ingest/_collections/fam-sliding-chunker.util.js +76 -0
- package/build/src/_modules/ingest/_collections/fam-ts-chunker.util.js +316 -0
- package/build/src/_modules/ingest/_models/interfaces/fam-ingest.interface.js +2 -0
- package/build/src/_modules/ingest/_services/fam-chunker.control-service.js +114 -0
- package/build/src/_modules/ingest/_services/fam-delta-compare.util.js +74 -0
- package/build/src/_modules/ingest/_services/fam-ingest-run.data-service.js +85 -0
- package/build/src/_modules/ingest/_services/fam-ingest.control-service.js +384 -0
- package/build/src/_modules/ingest/_services/fam-scan.control-service.js +211 -0
- package/build/src/_modules/ingest/index.js +46 -0
- package/build/src/_modules/mcp/_collections/fam-core-tools.const.js +186 -0
- package/build/src/_modules/mcp/_models/interfaces/fam-mcp.interface.js +31 -0
- package/build/src/_modules/mcp/_services/fam-capabilities-tool.service.js +111 -0
- package/build/src/_modules/mcp/_services/fam-capability-registry.service.js +1180 -0
- package/build/src/_modules/mcp/_services/fam-mcp-adapter.service.js +123 -0
- package/build/src/_modules/mcp/_services/fam-mcp-server.service.js +69 -0
- package/build/src/_modules/mcp/_services/fam-read-tool.service.js +99 -0
- package/build/src/_modules/mcp/_services/fam-write-tool.service.js +460 -0
- package/build/src/_modules/mcp/index.js +35 -0
- package/build/src/_modules/migration/_collections/fam-claude-mem-normalize.util.js +166 -0
- package/build/src/_modules/migration/_collections/fam-import-content-hash.util.js +38 -0
- package/build/src/_modules/migration/_collections/fam-target-mapping.util.js +90 -0
- package/build/src/_modules/migration/_enums/fam-claude-mem-source.type-enum.js +20 -0
- package/build/src/_modules/migration/_models/interfaces/fam-claude-mem.interface.js +26 -0
- package/build/src/_modules/migration/_services/fam-claude-mem-export-reader.service.js +134 -0
- package/build/src/_modules/migration/_services/fam-claude-mem-import.control-service.js +533 -0
- package/build/src/_modules/migration/_services/fam-claude-mem-sqlite-reader.service.js +144 -0
- package/build/src/_modules/migration/_services/fam-claude-mem-worker-reader.service.js +115 -0
- package/build/src/_modules/migration/_services/fam-import-dedup.data-service.js +102 -0
- package/build/src/_modules/migration/index.js +38 -0
- package/build/src/_modules/retrieval/_models/interfaces/fam-retrieval.interface.js +2 -0
- package/build/src/_modules/retrieval/_services/fam-retrieval-candidate.data-service.js +67 -0
- package/build/src/_modules/retrieval/_services/fam-retrieval-suggestions.util.js +182 -0
- package/build/src/_modules/retrieval/_services/fam-retrieval.control-service.js +282 -0
- package/build/src/_modules/retrieval/index.js +22 -0
- package/build/src/_modules/scope-reference/_collections/fam-fuzzy-match.util.js +86 -0
- package/build/src/_modules/scope-reference/_collections/fam-scope-normalize.util.js +47 -0
- package/build/src/_modules/scope-reference/_models/interfaces/fam-reference-resolution.interface.js +2 -0
- package/build/src/_modules/scope-reference/_models/interfaces/fam-resolution-trace.interface.js +2 -0
- package/build/src/_modules/scope-reference/_services/fam-reference.data-service.js +179 -0
- package/build/src/_modules/scope-reference/_services/fam-scope-resolver.control-service.js +473 -0
- package/build/src/_modules/scope-reference/_services/fam-scope.data-service.js +215 -0
- package/build/src/_modules/scope-reference/index.js +26 -0
- package/build/src/_routes/server/api/api.controller.js +400 -0
- package/build/src/_routes/server/client-app/client-app.control-service.js +132 -0
- package/build/src/_routes/server/client-app/client-app.controller.js +35 -0
- package/build/src/_routes/server/config/config.control-service.js +476 -0
- package/build/src/_routes/server/config/config.data-service.js +49 -0
- package/build/src/_routes/server/errors/errors.control-service.js +123 -0
- package/build/src/_routes/server/errors/errors.controller.js +65 -0
- package/build/src/_routes/server/errors/errors.data-service.js +80 -0
- package/build/src/_routes/server/server-status/server-status.control-service.js +19 -0
- package/build/src/_routes/server/server-status/server-status.controller.js +39 -0
- package/build/src/app.server.js +122 -0
- package/build/src/environments/environment.js +20 -0
- package/build/src/index.js +18 -0
- package/client-dist/chunk-GHKRM4SM.js +1 -0
- package/client-dist/chunk-LMTL7GA3.js +575 -0
- package/client-dist/index.html +17 -0
- package/client-dist/main-2KWB3QYK.js +2 -0
- package/client-dist/polyfills-HGDOEU5L.js +2 -0
- package/client-dist/styles-3J7JD5YE.css +1 -0
- package/package.json +96 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FAM_EmbeddingCost_ControlService = void 0;
|
|
4
|
+
const config_control_service_1 = require("../../../_routes/server/config/config.control-service");
|
|
5
|
+
const fam_embedding_pricing_const_1 = require("../_collections/fam-embedding-pricing.const");
|
|
6
|
+
/**
|
|
7
|
+
* `FAM_EmbeddingCost_ControlService` (SP-2.4, dsgn-006 §7) — **BFR-AM-007 projekt-lokális workaround**:
|
|
8
|
+
* embedding-költség hívás-típusonkénti aggregálása + a per-call cost-cap (`block-before-overrun` +
|
|
9
|
+
* `warn`). Singleton, in-memory aggregátor (a `run`-window az aktuális process-futás).
|
|
10
|
+
*
|
|
11
|
+
* **Token-becslés** (a Dynamo nem ad pontos token-számot a hívás ELŐTT): a szöveg-hossz / `CHARS_PER_TOKEN`
|
|
12
|
+
* heurisztika. Konzervatív (inkább túlbecsül), hogy a cap NE csússzon át túllépésbe. A tényleges
|
|
13
|
+
* token-fogyasztást a cost-event-ből (`recordActual`) korrigáljuk a hívás UTÁN.
|
|
14
|
+
*
|
|
15
|
+
* **Cost-cap precedencia:** a `embedding.costCapUsd` / `costCapWarnRatio` / `costWindow` a config-
|
|
16
|
+
* precedencián (`scope > table > global > builtin`) oldódik fel (dsgn-007 §4.4). `null` cap = nincs plafon.
|
|
17
|
+
*/
|
|
18
|
+
class FAM_EmbeddingCost_ControlService {
|
|
19
|
+
static _instance;
|
|
20
|
+
/** Az aktuális process-futás (`run`-window) hívás-típusonkénti aggregátuma. */
|
|
21
|
+
runBuckets = this.emptyBuckets();
|
|
22
|
+
static getInstance() {
|
|
23
|
+
if (!FAM_EmbeddingCost_ControlService._instance) {
|
|
24
|
+
FAM_EmbeddingCost_ControlService._instance = new FAM_EmbeddingCost_ControlService();
|
|
25
|
+
}
|
|
26
|
+
return FAM_EmbeddingCost_ControlService._instance;
|
|
27
|
+
}
|
|
28
|
+
// =========================================================================
|
|
29
|
+
// COST-CAP precheck (dsgn-006 §7) — a hívás ELŐTT
|
|
30
|
+
// =========================================================================
|
|
31
|
+
/**
|
|
32
|
+
* A cost-cap döntés a hívás ELŐTT (dsgn-006 §7). A becsült költséget (a `texts`-ből) hozzáadja az
|
|
33
|
+
* aktuális window-aggregátumhoz: ha `>= capUsd` → `blocked` (a `FAM_Embedding_ControlService`
|
|
34
|
+
* `FAM-EMB-COSTCAP-001`-et emittál); ha `>= capUsd * warnRatio` → `warn` (nem blokkol). `null` cap
|
|
35
|
+
* (vagy lokális provider) → soha nem blokkol/warn-ol.
|
|
36
|
+
*/
|
|
37
|
+
async precheck(context) {
|
|
38
|
+
const estimatedTokens = this.estimateTokens(context.texts);
|
|
39
|
+
const estimatedUsd = this.estimateUsd(context.provider, context.modelId, estimatedTokens);
|
|
40
|
+
const capUsd = await this.resolveCap('embedding.costCapUsd', context);
|
|
41
|
+
const warnRatio = (await this.resolveNumber('embedding.costCapWarnRatio', context)) ?? 0.8;
|
|
42
|
+
const windowKey = (await this.resolveWindow(context)) ?? 'run';
|
|
43
|
+
// Lokális provider (LM Studio / mock) → nincs USD-költség (dsgn-006 §7): soha nem blokkol.
|
|
44
|
+
const aggregatedUsd = this.windowAggregatedUsd(windowKey);
|
|
45
|
+
const projectedUsd = aggregatedUsd + estimatedUsd;
|
|
46
|
+
if (capUsd === null || capUsd <= 0) {
|
|
47
|
+
return {
|
|
48
|
+
blocked: false, warn: false, projectedUsd: projectedUsd, capUsd: capUsd,
|
|
49
|
+
reason: 'Nincs konfigurált embedding cost-plafon (embedding.costCapUsd = null).',
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
if (projectedUsd >= capUsd) {
|
|
53
|
+
return {
|
|
54
|
+
blocked: true, warn: false, projectedUsd: projectedUsd, capUsd: capUsd,
|
|
55
|
+
reason: `A következő embedding-hívás túllépné a(z) $${capUsd} plafont `
|
|
56
|
+
+ `(aggregált $${aggregatedUsd.toFixed(6)} + becsült $${estimatedUsd.toFixed(6)} = `
|
|
57
|
+
+ `$${projectedUsd.toFixed(6)}, window=${windowKey}).`,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
if (projectedUsd >= capUsd * warnRatio) {
|
|
61
|
+
return {
|
|
62
|
+
blocked: false, warn: true, projectedUsd: projectedUsd, capUsd: capUsd,
|
|
63
|
+
reason: `Az embedding-költség közelít a(z) $${capUsd} plafonhoz `
|
|
64
|
+
+ `(${(projectedUsd / capUsd * 100).toFixed(0)}%, window=${windowKey}).`,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
blocked: false, warn: false, projectedUsd: projectedUsd, capUsd: capUsd,
|
|
69
|
+
reason: 'A becsült költség a warn-küszöb alatt.',
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
// =========================================================================
|
|
73
|
+
// AGGREGÁCIÓ — a hívás UTÁN (cost-event / becslés)
|
|
74
|
+
// =========================================================================
|
|
75
|
+
/**
|
|
76
|
+
* A tényleges költség rögzítése a hívás UTÁN (a cost-event token-fogyasztásából, vagy ha az nem
|
|
77
|
+
* érkezik — pl. LM Studio/mock — a becsült tokenből). A `FAM_Embedding_ControlService` hívja a
|
|
78
|
+
* sikeres embed után. A `provider` lokális → `estimatedCostUsd=0`, csak token-telemetria.
|
|
79
|
+
*/
|
|
80
|
+
recordActual(set) {
|
|
81
|
+
const usd = this.estimateUsd(set.provider, set.modelId, set.tokens);
|
|
82
|
+
const bucket = this.runBuckets[set.callType];
|
|
83
|
+
bucket.tokens += set.tokens;
|
|
84
|
+
bucket.estimatedCostUsd += usd;
|
|
85
|
+
bucket.calls += 1;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* A teljes cost-stats (a `get_embedding_cost_stats` capability — dsgn-003 — kimenete). Hívás-
|
|
89
|
+
* típusonkénti bontás + összesítés. A `run`-window az aktuális process-futás aggregátuma.
|
|
90
|
+
*/
|
|
91
|
+
getStats() {
|
|
92
|
+
let totalTokens = 0;
|
|
93
|
+
let totalUsd = 0;
|
|
94
|
+
for (const callType of Object.keys(this.runBuckets)) {
|
|
95
|
+
totalTokens += this.runBuckets[callType].tokens;
|
|
96
|
+
totalUsd += this.runBuckets[callType].estimatedCostUsd;
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
byCallType: this.cloneBuckets(),
|
|
100
|
+
totalTokens: totalTokens,
|
|
101
|
+
totalEstimatedCostUsd: totalUsd,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/** Az aggregátum nullázása (teszt-izoláció + a `run`-window reset). */
|
|
105
|
+
reset() {
|
|
106
|
+
this.runBuckets = this.emptyBuckets();
|
|
107
|
+
}
|
|
108
|
+
// =========================================================================
|
|
109
|
+
// becslés + helpers
|
|
110
|
+
// =========================================================================
|
|
111
|
+
/** Token-becslés a szövegek összhosszából (~4 char/token heurisztika; konzervatív felfelé). */
|
|
112
|
+
estimateTokens(texts) {
|
|
113
|
+
const totalChars = texts.reduce((sum, text) => sum + (text?.length ?? 0), 0);
|
|
114
|
+
return Math.ceil(totalChars / fam_embedding_pricing_const_1.FAM_EMBEDDING_CHARS_PER_TOKEN);
|
|
115
|
+
}
|
|
116
|
+
/** A becsült USD egy adott token-számra (lokális provider → 0; OpenAI → ár-tábla / fallback). */
|
|
117
|
+
estimateUsd(provider, modelId, tokens) {
|
|
118
|
+
if (provider !== 'openai') {
|
|
119
|
+
return 0;
|
|
120
|
+
}
|
|
121
|
+
const pricePer1k = fam_embedding_pricing_const_1.FAM_EMBEDDING_PRICE_PER_1K_TOKENS[modelId] ?? fam_embedding_pricing_const_1.FAM_EMBEDDING_DEFAULT_OPENAI_PRICE_PER_1K;
|
|
122
|
+
return (tokens / 1000) * pricePer1k;
|
|
123
|
+
}
|
|
124
|
+
/** Az adott window aggregált USD-je. MVP1: a `run`-window az in-memory aggregátum; `day`/`total`
|
|
125
|
+
* szintén ezt használja (a perzistált window-aggregátor a bedrock cost-store-ral jön — BFR-AM-007). */
|
|
126
|
+
windowAggregatedUsd(_window) {
|
|
127
|
+
let total = 0;
|
|
128
|
+
for (const callType of Object.keys(this.runBuckets)) {
|
|
129
|
+
total += this.runBuckets[callType].estimatedCostUsd;
|
|
130
|
+
}
|
|
131
|
+
return total;
|
|
132
|
+
}
|
|
133
|
+
/** A `costCapUsd` (number-or-null) feloldása a config-precedencián. */
|
|
134
|
+
async resolveCap(key, context) {
|
|
135
|
+
const resolved = await config_control_service_1.FAM_Config_ControlService.getInstance().resolve(key, {
|
|
136
|
+
table: context.table, scopePath: context.scopePath,
|
|
137
|
+
});
|
|
138
|
+
return typeof resolved.value === 'number' ? resolved.value : null;
|
|
139
|
+
}
|
|
140
|
+
/** Egy number config feloldása (warn-ratio); ha nem number → null. */
|
|
141
|
+
async resolveNumber(key, context) {
|
|
142
|
+
const resolved = await config_control_service_1.FAM_Config_ControlService.getInstance().resolve(key, {
|
|
143
|
+
table: context.table, scopePath: context.scopePath,
|
|
144
|
+
});
|
|
145
|
+
return typeof resolved.value === 'number' ? resolved.value : null;
|
|
146
|
+
}
|
|
147
|
+
/** A `costWindow` feloldása (string enum). */
|
|
148
|
+
async resolveWindow(context) {
|
|
149
|
+
const resolved = await config_control_service_1.FAM_Config_ControlService.getInstance().resolve('embedding.costWindow', {
|
|
150
|
+
table: context.table, scopePath: context.scopePath,
|
|
151
|
+
});
|
|
152
|
+
if (resolved.value === 'run' || resolved.value === 'day' || resolved.value === 'total') {
|
|
153
|
+
return resolved.value;
|
|
154
|
+
}
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
/** Üres aggregátum minden hívás-típusra. */
|
|
158
|
+
emptyBuckets() {
|
|
159
|
+
return {
|
|
160
|
+
'embed-write': { tokens: 0, estimatedCostUsd: 0, calls: 0 },
|
|
161
|
+
'embed-scan': { tokens: 0, estimatedCostUsd: 0, calls: 0 },
|
|
162
|
+
'embed-re-embed': { tokens: 0, estimatedCostUsd: 0, calls: 0 },
|
|
163
|
+
'embed-query': { tokens: 0, estimatedCostUsd: 0, calls: 0 },
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/** Az aggregátum másolata (a `getStats` ne adjon élő referenciát). */
|
|
167
|
+
cloneBuckets() {
|
|
168
|
+
const result = this.emptyBuckets();
|
|
169
|
+
for (const callType of Object.keys(this.runBuckets)) {
|
|
170
|
+
result[callType] = { ...this.runBuckets[callType] };
|
|
171
|
+
}
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
exports.FAM_EmbeddingCost_ControlService = FAM_EmbeddingCost_ControlService;
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FAM_EmbeddingPipeline_ControlService = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
const crypto = tslib_1.__importStar(require("crypto"));
|
|
6
|
+
const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
|
|
7
|
+
const error_codes_const_1 = require("../../../_collections/error-codes.const");
|
|
8
|
+
const errors_control_service_1 = require("../../../_routes/server/errors/errors.control-service");
|
|
9
|
+
const fam_store_registry_const_1 = require("../_collections/fam-store-registry.const");
|
|
10
|
+
const fam_embedding_control_service_1 = require("./fam-embedding.control-service");
|
|
11
|
+
const fam_entry_data_service_1 = require("./fam-entry.data-service");
|
|
12
|
+
const fam_vector_search_control_service_1 = require("./fam-vector-search.control-service");
|
|
13
|
+
/**
|
|
14
|
+
* `FAM_EmbeddingPipeline_ControlService` (SP-2.2 auto-embed + SP-2.4 re-embed) — az embedding-réteg
|
|
15
|
+
* írás-oldali orchestrátora. Singleton. Két fő belépési pont:
|
|
16
|
+
*
|
|
17
|
+
* 1. **`embedAndPersist`** (auto-embed; dsgn-006 §3) — a write/scan path mentés UTÁN ezt hívja egy
|
|
18
|
+
* entry-re (MP-4/MP-5): `contentHash` → embed (provider-agnosztikus) → `contentVector` + `embeddingModel`
|
|
19
|
+
* + `embeddingStatus='completed'` atomikus `$set` → LVS pool upsert. Hiba → `embeddingStatus='error'`
|
|
20
|
+
* + `FAM_Error` (dsgn-008), az entry NEM kerül a keresésbe.
|
|
21
|
+
* 2. **`reEmbedTable` / `reEmbedScope`** (re-embed; dsgn-006 §5) — modell-/provider-váltáskor a meglévő
|
|
22
|
+
* vektorok frissítése az aktuális modellre, kötegelten (flow-limit), `pending→completed/error`
|
|
23
|
+
* lifecycle-lel, LVS pool re-hidratálással. Eredmény `{ reembedded, skipped, failed }`.
|
|
24
|
+
*
|
|
25
|
+
* **AUTO-EMBED-ON-SAVE DÖNTÉS (dsgn-006 §3, lásd a modul-README-t):** a Dynamo `DyNTS_OAI_VectorDataService.saveData`
|
|
26
|
+
* auto-vektorizál a `vectorizedFrom` annotációból, DE hardwired OpenAI-ra (`DyFM_OAI_Settings` + OpenAI-kliens),
|
|
27
|
+
* és NINCS globális provider-agnosztikus embedding-hook a `DyNTS_DataService.saveData`-ban. Ezért a FAM az
|
|
28
|
+
* embeddinget EXPLICIT pipeline-on (ez a service) végzi — a `FAM_Embedding_ControlService` provider-dispatch-én
|
|
29
|
+
* át (OpenAI/LM Studio/mock) — NEM a Dynamo auto-vektorizáláson. Így a `vectorizedFrom` annotáció a forrás-mező-
|
|
30
|
+
* jelölés szerepét tölti be (a content-kompozícióhoz), az embedding-hívás viszont a FAM kapuján megy.
|
|
31
|
+
*/
|
|
32
|
+
class FAM_EmbeddingPipeline_ControlService {
|
|
33
|
+
static _instance;
|
|
34
|
+
issuer = 'FAM_EmbeddingPipeline_ControlService';
|
|
35
|
+
/** A re-embed flow-limit felső korlátja (CCAP MAX 200; dsgn-007 §4.7). */
|
|
36
|
+
maxFlowLimit = 200;
|
|
37
|
+
static getInstance() {
|
|
38
|
+
if (!FAM_EmbeddingPipeline_ControlService._instance) {
|
|
39
|
+
FAM_EmbeddingPipeline_ControlService._instance = new FAM_EmbeddingPipeline_ControlService();
|
|
40
|
+
}
|
|
41
|
+
return FAM_EmbeddingPipeline_ControlService._instance;
|
|
42
|
+
}
|
|
43
|
+
// =========================================================================
|
|
44
|
+
// AUTO-EMBED (dsgn-006 §3) — write/scan path mentés után
|
|
45
|
+
// =========================================================================
|
|
46
|
+
/**
|
|
47
|
+
* Egy entry embeddelése + persist + pool-upsert (dsgn-006 §3). A write/scan path (MP-4/MP-5) hívja a
|
|
48
|
+
* Mongo-mentés UTÁN (az entry már létezik `_id`-vel). A flow: content-kompozíció (`vectorizedFrom`) →
|
|
49
|
+
* `contentHash` → embed → atomikus `$set` (`contentVector`+`embeddingModel`+`completed`) → pool upsert.
|
|
50
|
+
* Hiba → `embeddingStatus='error'` + `FAM_Error` (a hívónak visszaadott eredmény jelzi a státuszt).
|
|
51
|
+
*/
|
|
52
|
+
async embedAndPersist(set) {
|
|
53
|
+
const registryEntry = fam_store_registry_const_1.FAM_StoreRegistry_Util.getEntry(set.table);
|
|
54
|
+
if (!registryEntry || !set.entry._id) {
|
|
55
|
+
return { status: 'error', dims: 0 };
|
|
56
|
+
}
|
|
57
|
+
const dataService = new fam_entry_data_service_1.FAM_Entry_DataService({ dataParams: registryEntry.dataParams, issuer: this.issuer });
|
|
58
|
+
const composed = this.composeVectorizedText(set.entry, registryEntry.vectorizedFrom);
|
|
59
|
+
const contentHash = this.sha256(composed);
|
|
60
|
+
if (!composed.trim().length) {
|
|
61
|
+
// Nincs vektorizálható content → marad pending (a write-validáció a content-kötelezőséget
|
|
62
|
+
// külön kezeli; itt csak jelezzük, hogy nincs mit embeddelni).
|
|
63
|
+
return { status: 'error', dims: 0 };
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
const vectors = await fam_embedding_control_service_1.FAM_Embedding_ControlService.getInstance().embedTexts({
|
|
67
|
+
texts: [composed],
|
|
68
|
+
table: set.table,
|
|
69
|
+
scopePath: set.entry.scopePath,
|
|
70
|
+
callType: set.callType ?? 'embed-write',
|
|
71
|
+
issuer: this.issuer,
|
|
72
|
+
});
|
|
73
|
+
const vector = vectors[0] ?? [];
|
|
74
|
+
const resolved = await fam_embedding_control_service_1.FAM_Embedding_ControlService.getInstance().resolveProvider({
|
|
75
|
+
table: set.table, scopePath: set.entry.scopePath,
|
|
76
|
+
});
|
|
77
|
+
await dataService.setVector({
|
|
78
|
+
id: set.entry._id,
|
|
79
|
+
contentVector: vector,
|
|
80
|
+
embeddingModel: resolved.modelId,
|
|
81
|
+
embeddingStatus: 'completed',
|
|
82
|
+
contentHash: contentHash,
|
|
83
|
+
});
|
|
84
|
+
fam_vector_search_control_service_1.FAM_VectorSearch_ControlService.getInstance().upsertVector({ table: set.table, id: set.entry._id, vector: vector });
|
|
85
|
+
return { status: 'completed', dims: vector.length };
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
// No-silent-failure: az entry `error`-státuszba kerül + a hiba persistált; a pool-ból kivesszük.
|
|
89
|
+
await dataService.setEmbeddingStatus(set.entry._id, 'error');
|
|
90
|
+
fam_vector_search_control_service_1.FAM_VectorSearch_ControlService.getInstance().removeVector(set.table, set.entry._id);
|
|
91
|
+
await errors_control_service_1.FAM_Error_ControlService.getInstance().emit({
|
|
92
|
+
errorCode: error_codes_const_1.FAM_ERROR_CODES.embProviderUnreachable,
|
|
93
|
+
message: `Az entry embeddelése sikertelen (table='${set.table}', id='${set.entry._id}').`,
|
|
94
|
+
cause: error,
|
|
95
|
+
context: { operation: 'embed-and-persist', table: set.table },
|
|
96
|
+
});
|
|
97
|
+
return { status: 'error', dims: 0 };
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
// =========================================================================
|
|
101
|
+
// RE-EMBED (dsgn-006 §5) — modell-/provider-váltás
|
|
102
|
+
// =========================================================================
|
|
103
|
+
/**
|
|
104
|
+
* Egy teljes tár re-embed-je (dsgn-006 §5). A `completed`/`error`/`pending` cél-entry-ket (`!_deleted`)
|
|
105
|
+
* kötegelten (flow-limit) újra-embeddeli az AKTUÁLIS modellre: `pending` → `embed` → `contentVector` +
|
|
106
|
+
* `embeddingModel` frissítés → `completed`/`error` → LVS pool upsert. Az `addedBy='re-embed'` provenance
|
|
107
|
+
* a hívó write-path-é (itt a vektor-frissítésre fókuszálunk). Eredmény `{ reembedded, skipped, failed }`.
|
|
108
|
+
*/
|
|
109
|
+
async reEmbedTable(table, flowLimit) {
|
|
110
|
+
return this.reEmbed({ table: table, flowLimit: flowLimit });
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Egy tár adott scope-ágának re-embed-je (dsgn-006 §5). A scope-prefiltert MP-3/MP-5 adja (a
|
|
114
|
+
* `scopeId`-halmaz); itt a `scopePath` LEVÉL scopeId-jára szűkítünk (a `scopePath.scopeId` mező
|
|
115
|
+
* indexelt — a write-path canonical scopePath-ot ír az entry-re).
|
|
116
|
+
*/
|
|
117
|
+
async reEmbedScope(table, scopePath, flowLimit) {
|
|
118
|
+
return this.reEmbed({ table: table, scopePath: scopePath, flowLimit: flowLimit });
|
|
119
|
+
}
|
|
120
|
+
/** A re-embed mag (table + opc. scope-szűkítés). Re-hidratálja az érintett pool-t a frissített vektorral. */
|
|
121
|
+
async reEmbed(set) {
|
|
122
|
+
const registryEntry = fam_store_registry_const_1.FAM_StoreRegistry_Util.getEntry(set.table);
|
|
123
|
+
const result = {
|
|
124
|
+
table: set.table,
|
|
125
|
+
scopePath: set.scopePath?.map((ref) => ref.canonicalName).join('>'),
|
|
126
|
+
total: 0, reembedded: 0, skipped: 0, failed: 0,
|
|
127
|
+
};
|
|
128
|
+
if (!registryEntry) {
|
|
129
|
+
return result;
|
|
130
|
+
}
|
|
131
|
+
const dataService = new fam_entry_data_service_1.FAM_Entry_DataService({ dataParams: registryEntry.dataParams, issuer: this.issuer });
|
|
132
|
+
const filter = this.buildScopeFilter(set.scopePath);
|
|
133
|
+
const targets = await dataService.findHydratableList(filter);
|
|
134
|
+
result.total = targets.length;
|
|
135
|
+
const flowLimit = Math.min(Math.max(set.flowLimit ?? 20, 1), this.maxFlowLimit);
|
|
136
|
+
// Kötegelt feldolgozás (flow-limit; rate-/token-limit védelem, dsgn-006 §5).
|
|
137
|
+
for (let i = 0; i < targets.length; i += flowLimit) {
|
|
138
|
+
const batch = targets.slice(i, i + flowLimit);
|
|
139
|
+
await Promise.all(batch.map((entry) => this.reEmbedOne(set.table, entry, registryEntry, dataService, result)));
|
|
140
|
+
}
|
|
141
|
+
fsm_dynamo_1.DyFM_Log.log(`[FAM re-embed] '${set.table}'${result.scopePath ? ` (scope=${result.scopePath})` : ''}: `
|
|
142
|
+
+ `total=${result.total} reembedded=${result.reembedded} skipped=${result.skipped} failed=${result.failed}`);
|
|
143
|
+
return result;
|
|
144
|
+
}
|
|
145
|
+
/** Egy entry re-embed-je (pending → embed → completed/error → pool upsert). Mutálja a `result`-ot. */
|
|
146
|
+
async reEmbedOne(table, entry, registryEntry, dataService, result) {
|
|
147
|
+
if (!entry._id) {
|
|
148
|
+
result.skipped++;
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const composed = this.composeVectorizedText(entry, registryEntry.vectorizedFrom);
|
|
152
|
+
if (!composed.trim().length) {
|
|
153
|
+
result.skipped++;
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
await dataService.setEmbeddingStatus(entry._id, 'pending');
|
|
157
|
+
const outcome = await this.embedAndPersist({ table: table, entry: entry, callType: 'embed-re-embed' });
|
|
158
|
+
if (outcome.status === 'completed') {
|
|
159
|
+
result.reembedded++;
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
result.failed++;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
// =========================================================================
|
|
166
|
+
// content-kompozíció + hash + scope-filter
|
|
167
|
+
// =========================================================================
|
|
168
|
+
/**
|
|
169
|
+
* A vektorizálandó szöveg összeállítása a `vectorizedFrom` forrás-mezőkből (dsgn-001 §8, dsgn-006 §3).
|
|
170
|
+
* Pl. knowledge: `content` + `subject` + `description`. A nem-string / üres mezőket kihagyja; a
|
|
171
|
+
* mezőket sortöréssel fűzi (a szemantikai jel maximalizálásához). A `FAM_Entry` dinamikus mező-
|
|
172
|
+
* olvasását `Reflect.get`-tel végzi (a per-tár mezők — pl. `subject` — nincsenek a bázis-shape-en).
|
|
173
|
+
*/
|
|
174
|
+
composeVectorizedText(entry, vectorizedFrom) {
|
|
175
|
+
const parts = [];
|
|
176
|
+
for (const field of vectorizedFrom) {
|
|
177
|
+
const value = Reflect.get(entry, field);
|
|
178
|
+
if (typeof value === 'string' && value.trim().length) {
|
|
179
|
+
parts.push(value.trim());
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return parts.join('\n');
|
|
183
|
+
}
|
|
184
|
+
/** SHA-256(normalizált szöveg) — a delta-detection contentHash-e (dsgn-001 / dsgn-004 §4). */
|
|
185
|
+
sha256(text) {
|
|
186
|
+
return crypto.createHash('sha256').update(this.normalize(text)).digest('hex');
|
|
187
|
+
}
|
|
188
|
+
/** Egyszerű normalizálás a hash-hez (whitespace-tömörítés + trim) — a delta-konzisztenciához. */
|
|
189
|
+
normalize(text) {
|
|
190
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
191
|
+
}
|
|
192
|
+
/** A scope-szűkítő filter (a `scopePath` LEVÉL scopeId-jára). Üres scope → teljes tár (`!_deleted`). */
|
|
193
|
+
buildScopeFilter(scopePath) {
|
|
194
|
+
if (scopePath && scopePath.length) {
|
|
195
|
+
const leaf = scopePath[scopePath.length - 1];
|
|
196
|
+
const filter = { 'scopePath.scopeId': leaf.scopeId };
|
|
197
|
+
return filter;
|
|
198
|
+
}
|
|
199
|
+
return {};
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
exports.FAM_EmbeddingPipeline_ControlService = FAM_EmbeddingPipeline_ControlService;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FAM_EmbeddingPreset_ControlService = void 0;
|
|
4
|
+
const config_control_service_1 = require("../../../_routes/server/config/config.control-service");
|
|
5
|
+
const fam_embedding_control_service_1 = require("./fam-embedding.control-service");
|
|
6
|
+
/**
|
|
7
|
+
* `FAM_EmbeddingPreset_ControlService` (SP-2.3, dsgn-006 §6) — az embedding-presetek + per-tár/scope
|
|
8
|
+
* modell-override embedding-lencséje. Singleton. A preset-bundle SSOT-ja a MP-7 `CONFIG_PRESETS`; az
|
|
9
|
+
* `applyPreset` a MP-7 `FAM_Config_ControlService.applyPreset`-re delegál (rendes `FAM_Config` rekordok,
|
|
10
|
+
* `setBy='preset'`, atomikus + validált — dsgn-007 §5). EZ a service a `fam init` / CLI (MP-9/12) felé
|
|
11
|
+
* adott embedding-orientált belépési pont (preset-felajánlás + override-helper).
|
|
12
|
+
*
|
|
13
|
+
* **A dimenzió-eltérés → re-embed jelzés** (acceptance #6): a preset/override dimenzió-váltása (pl.
|
|
14
|
+
* global OpenAI 3072 → table LM Studio 768) re-embedet igényel — a `wouldRequireReEmbed` ezt jelzi a
|
|
15
|
+
* hívónak (a tényleges re-embedet a `FAM_EmbeddingPipeline_ControlService.reEmbedTable` futtatja).
|
|
16
|
+
*/
|
|
17
|
+
class FAM_EmbeddingPreset_ControlService {
|
|
18
|
+
static _instance;
|
|
19
|
+
issuer = 'FAM_EmbeddingPreset_ControlService';
|
|
20
|
+
static getInstance() {
|
|
21
|
+
if (!FAM_EmbeddingPreset_ControlService._instance) {
|
|
22
|
+
FAM_EmbeddingPreset_ControlService._instance = new FAM_EmbeddingPreset_ControlService();
|
|
23
|
+
}
|
|
24
|
+
return FAM_EmbeddingPreset_ControlService._instance;
|
|
25
|
+
}
|
|
26
|
+
/** A felajánlható beépített preset-nevek (a `fam init` / `create-config` ezt listázza — dsgn-006 §6). */
|
|
27
|
+
availablePresets() {
|
|
28
|
+
return ['openai-recommended', 'lmstudio-recommended'];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Egy preset alkalmazása (MP-7-re delegálva). `table` nélkül → table-szint kötelező (a MP-7 validál);
|
|
32
|
+
* `scopeId`-val → scope-szint. Idempotens + atomikus (egy invalid kulcs → semmi nem íródik, dsgn-007 §5).
|
|
33
|
+
* Pl. a nagy `codebase` tár `lmstudio-recommended`-et kaphat table-szinten, míg a global OpenAI marad.
|
|
34
|
+
*/
|
|
35
|
+
async applyPreset(set) {
|
|
36
|
+
return config_control_service_1.FAM_Config_ControlService.getInstance().applyPreset(set.name, {
|
|
37
|
+
table: set.table,
|
|
38
|
+
scopeId: set.scopeId,
|
|
39
|
+
issuer: set.issuer ?? this.issuer,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* A jelenleg feloldott embedding-provider egy kontextusra (a `get_embedding_config` capability —
|
|
44
|
+
* dsgn-003). A per-tár/scope override-ot a `FAM_Embedding_ControlService.resolveProvider` adja vissza.
|
|
45
|
+
*/
|
|
46
|
+
async getResolvedProvider(set) {
|
|
47
|
+
return fam_embedding_control_service_1.FAM_Embedding_ControlService.getInstance().resolveProvider({ table: set.table, scopePath: set.scopePath });
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Re-embed szükséges-e egy tervezett modell-/dimenzió-váltáshoz (acceptance #6)? A jelenlegi
|
|
51
|
+
* feloldott dimenziót összeveti a tervezett `targetDims`-szel: eltérés → `true` (a pool nem keverhet
|
|
52
|
+
* eltérő dimenziót; a hívónak re-embedet kell futtatnia). `null` (natív) dim → nem dönthető el
|
|
53
|
+
* statikusan (`false`, a tényleges modell-dim a hívás-időpontban derül ki).
|
|
54
|
+
*/
|
|
55
|
+
async wouldRequireReEmbed(set) {
|
|
56
|
+
if (set.targetDims === null) {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
const current = await this.getResolvedProvider({ table: set.table, scopePath: set.scopePath });
|
|
60
|
+
if (current.dims === null) {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
return current.dims !== set.targetDims;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
exports.FAM_EmbeddingPreset_ControlService = FAM_EmbeddingPreset_ControlService;
|