@futdevpro/nts-dynamo 1.15.58 → 1.15.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dynamo/logs/cicd-pipeline/output.log +1622 -1705
- package/.dynamo/logs/cicd-pipeline/status.json +30 -30
- package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.d.ts.map +1 -1
- package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.js +2 -2
- package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.js.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-mock.service.d.ts +1 -1
- package/build/_modules/ai/_services/ai-embedding-mock.service.d.ts.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-mock.service.js.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-provider.registry.d.ts.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-provider.registry.js.map +1 -1
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.d.ts +1 -1
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.d.ts.map +1 -1
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.js +3 -3
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.js.map +1 -1
- package/build/_modules/ai/index.d.ts +2 -0
- package/build/_modules/ai/index.d.ts.map +1 -1
- package/build/_modules/ai/index.js +4 -0
- package/build/_modules/ai/index.js.map +1 -1
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.d.ts +17 -17
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.d.ts.map +1 -1
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.js +21 -21
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.js.map +1 -1
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.d.ts +4 -4
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.d.ts.map +1 -1
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.js +5 -5
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.js.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.d.ts +4 -4
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.d.ts.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.js +4 -4
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.js.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.d.ts +6 -6
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.d.ts.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.js +5 -5
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.js.map +1 -1
- package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.d.ts +1 -1
- package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.js +1 -1
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.d.ts +4 -4
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.d.ts.map +1 -1
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.js +6 -6
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.js.map +1 -1
- package/build/_modules/mcp/_services/dynts-mcp.adapter.d.ts +11 -11
- package/build/_modules/mcp/_services/dynts-mcp.adapter.d.ts.map +1 -1
- package/build/_modules/mcp/_services/dynts-mcp.adapter.js +16 -11
- package/build/_modules/mcp/_services/dynts-mcp.adapter.js.map +1 -1
- package/build/_modules/mcp/index.js +1 -1
- package/build/_modules/mcp/index.js.map +1 -1
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.d.ts +3 -3
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.d.ts.map +1 -1
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.js +4 -4
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.js.map +1 -1
- package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.d.ts.map +1 -1
- package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.js +9 -0
- package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.js.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.d.ts +3 -3
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.d.ts.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.js +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.js.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.d.ts +7 -7
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.d.ts.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.js +2 -2
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.js.map +1 -1
- package/package.json +1 -1
- package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +39 -7
- package/src/_modules/ai/_services/ai-embedding-mock.service.ts +18 -4
- package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +4 -0
- package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +26 -5
- package/src/_modules/ai/index.ts +5 -0
- package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +145 -130
- package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +131 -120
- package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +6 -5
- package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +35 -35
- package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +9 -5
- package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +11 -11
- package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +19 -17
- package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +1 -1
- package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +123 -114
- package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +44 -39
- package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +114 -103
- package/src/_modules/mcp/index.ts +1 -1
- package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +5 -4
- package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +0 -2
- package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +19 -13
- package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +37 -21
- package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +11 -6
- package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +17 -14
|
@@ -5,8 +5,7 @@ import Database from 'better-sqlite3';
|
|
|
5
5
|
import { DyFM_Error } from '@futdevpro/fsm-dynamo';
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
|
-
|
|
9
|
-
DyNTS_SqliteReadOptions,
|
|
8
|
+
DyNTS_SqliteColumnInfo
|
|
10
9
|
} from '../_models/interfaces/dynts-sqlite-reader.interface';
|
|
11
10
|
|
|
12
11
|
/**
|
|
@@ -29,156 +28,168 @@ import {
|
|
|
29
28
|
*/
|
|
30
29
|
export class DyNTS_Sqlite_Reader_Util {
|
|
31
30
|
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
/** A hibák issuer-e (a strukturált DyFM_Error-okhoz). */
|
|
32
|
+
private static readonly issuer: string = 'DyNTS_Sqlite_Reader_Util';
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
|
|
34
|
+
/** A `query()` SELECT-only guard regex-e: a (trimmelt, lowercase) SQL `select`/`with`-tel kezdődik. */
|
|
35
|
+
private static readonly SELECT_ONLY: RegExp = /^\s*(select|with)\b/i;
|
|
37
36
|
|
|
38
|
-
|
|
37
|
+
/**
|
|
39
38
|
* Egy tábla ÖSSZES sorának read-only olvasása (`SELECT * FROM <table>`). A `table` nevet
|
|
40
39
|
* `sqlite_master` ellen validáljuk (csak létező táblanév fut, így nincs SQL-injekciós felület a
|
|
41
40
|
* tábla-néven). Nem-létező tábla → strukturált hiba (NEM néma-[]).
|
|
42
41
|
*/
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
});
|
|
51
|
-
}
|
|
52
|
-
// A `table` itt már egy igazolt, létező táblanév (sqlite_master), nem nyers user-input.
|
|
53
|
-
return db.prepare(`SELECT * FROM "${table}"`).all();
|
|
42
|
+
public static readTable(dbPath: string, table: string): unknown[] {
|
|
43
|
+
return DyNTS_Sqlite_Reader_Util.withDb(dbPath, (db) => {
|
|
44
|
+
if (!DyNTS_Sqlite_Reader_Util.tableExists(db, table)) {
|
|
45
|
+
throw new DyFM_Error({
|
|
46
|
+
errorCode: 'DyNTS-SQLITE-READ-002',
|
|
47
|
+
message: `Table '${table}' does not exist in the SQLite DB: '${dbPath}'.`,
|
|
48
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
54
49
|
});
|
|
55
|
-
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// A `table` itt már egy igazolt, létező táblanév (sqlite_master), nem nyers user-input.
|
|
53
|
+
return db.prepare(`SELECT * FROM "${table}"`).all();
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
/**
|
|
58
58
|
* Egy tetszőleges **SELECT-only** lekérdezés read-only futtatása. A guard elutasít minden nem-SELECT
|
|
59
59
|
* (insert/update/delete/drop/attach/pragma-write/…) SQL-t → `DyNTS-SQLITE-READONLY-001`. A
|
|
60
60
|
* `params` opcionális prepared-statement paraméterek (pozícionális `?` vagy named `@x`).
|
|
61
61
|
*/
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
return DyNTS_Sqlite_Reader_Util.withDb(dbPath, (db) => {
|
|
65
|
-
const statement: Database.Statement = db.prepare(sql);
|
|
66
|
-
// A `better-sqlite3` `.all()` csak read-statement-en ad sort; write-statement-en dobna —
|
|
67
|
-
// a SELECT-only guard ezt már a forrás-rétegben kizárja (defenzív kettősség).
|
|
68
|
-
return params === undefined ? statement.all() : statement.all(params as never);
|
|
69
|
-
});
|
|
70
|
-
}
|
|
62
|
+
public static query(dbPath: string, sql: string, params?: unknown[] | { [key: string]: unknown }): unknown[] {
|
|
63
|
+
DyNTS_Sqlite_Reader_Util.assertSelectOnly(sql);
|
|
71
64
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
65
|
+
return DyNTS_Sqlite_Reader_Util.withDb(dbPath, (db) => {
|
|
66
|
+
const statement: Database.Statement = db.prepare(sql);
|
|
67
|
+
|
|
68
|
+
// A `better-sqlite3` `.all()` csak read-statement-en ad sort; write-statement-en dobna —
|
|
69
|
+
// a SELECT-only guard ezt már a forrás-rétegben kizárja (defenzív kettősség).
|
|
70
|
+
return params === undefined ? statement.all() : statement.all(params);
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** A DB ÖSSZES (nem-belső) táblanevének read-only listája (`sqlite_master`, ABC-sorrendben). */
|
|
75
|
+
public static listTables(dbPath: string): string[] {
|
|
76
|
+
return DyNTS_Sqlite_Reader_Util.withDb(dbPath, (db) => {
|
|
77
|
+
const rows: { name: string }[] = db.prepare(
|
|
78
|
+
'SELECT name FROM sqlite_master WHERE type=\'table\' AND name NOT LIKE \'sqlite_%\' ORDER BY name',
|
|
79
|
+
).all() as { name: string }[];
|
|
81
80
|
|
|
82
|
-
|
|
81
|
+
return rows.map((row) => row.name);
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
83
86
|
* Egy tábla séma-leírása (`PRAGMA table_info(<table>)` — a SQLite read-only introspekciója). A
|
|
84
87
|
* `table` nevet `sqlite_master` ellen validáljuk (nem-létező → strukturált hiba). Az oszlop-info
|
|
85
88
|
* (`cid`/`name`/`type`/`notNull`/`defaultValue`/`primaryKey`) normalizált shape-ben tér vissza.
|
|
86
89
|
*/
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
});
|
|
95
|
-
}
|
|
96
|
-
const rows: PragmaColumnRow[] = db.pragma(`table_info("${table}")`) as PragmaColumnRow[];
|
|
97
|
-
return rows.map((row) => ({
|
|
98
|
-
cid: row.cid,
|
|
99
|
-
name: row.name,
|
|
100
|
-
type: row.type,
|
|
101
|
-
notNull: row.notnull === 1,
|
|
102
|
-
defaultValue: row.dflt_value,
|
|
103
|
-
primaryKey: row.pk > 0,
|
|
104
|
-
}));
|
|
90
|
+
public static getSchema(dbPath: string, table: string): DyNTS_SqliteColumnInfo[] {
|
|
91
|
+
return DyNTS_Sqlite_Reader_Util.withDb(dbPath, (db) => {
|
|
92
|
+
if (!DyNTS_Sqlite_Reader_Util.tableExists(db, table)) {
|
|
93
|
+
throw new DyFM_Error({
|
|
94
|
+
errorCode: 'DyNTS-SQLITE-READ-002',
|
|
95
|
+
message: `Table '${table}' does not exist in the SQLite DB: '${dbPath}'.`,
|
|
96
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
105
97
|
});
|
|
106
|
-
|
|
98
|
+
}
|
|
99
|
+
const rows: PragmaColumnRow[] = db.pragma(`table_info("${table}")`) as PragmaColumnRow[];
|
|
107
100
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
101
|
+
return rows.map((row) => ({
|
|
102
|
+
cid: row.cid,
|
|
103
|
+
name: row.name,
|
|
104
|
+
type: row.type,
|
|
105
|
+
notNull: row.notnull === 1,
|
|
106
|
+
defaultValue: row.dflt_value,
|
|
107
|
+
primaryKey: row.pk > 0,
|
|
108
|
+
}));
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
111
|
|
|
112
|
-
|
|
112
|
+
// =========================================================================
|
|
113
|
+
// Belső segédek (read-only megnyitás + guard-ok)
|
|
114
|
+
// =========================================================================
|
|
115
|
+
|
|
116
|
+
/**
|
|
113
117
|
* A DB read-only megnyitása + a callback futtatása + GARANTÁLT zárás (try/finally). A `path`
|
|
114
118
|
* validálva (üres/nem-létező → strukturált hiba); a megnyitás-/olvasás-hibát strukturált
|
|
115
119
|
* `DyFM_Error`-rá fordítjuk (a már-DyFM_Error-t NEM csomagoljuk újra). A `readonly:true` az
|
|
116
120
|
* első védvonal (a SQLite-engine elutasít minden írást).
|
|
117
121
|
*/
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
122
|
+
private static withDb<T>(dbPath: string, work: (db: Database.Database) => T): T {
|
|
123
|
+
if (!dbPath?.trim().length) {
|
|
124
|
+
throw new DyFM_Error({
|
|
125
|
+
errorCode: 'DyNTS-SQLITE-READ-001',
|
|
126
|
+
message: 'A SQLite read requires a non-empty `dbPath` (path to the `.db` file).',
|
|
127
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (!fs.existsSync(dbPath)) {
|
|
132
|
+
throw new DyFM_Error({
|
|
133
|
+
errorCode: 'DyNTS-SQLITE-READ-001',
|
|
134
|
+
message: `The SQLite DB file was not found: '${dbPath}'.`,
|
|
135
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
let db: Database.Database | undefined = undefined;
|
|
139
|
+
|
|
140
|
+
try {
|
|
141
|
+
db = new Database(dbPath, { readonly: true, fileMustExist: true });
|
|
142
|
+
|
|
143
|
+
return work(db);
|
|
144
|
+
} catch (error) {
|
|
145
|
+
if (error instanceof DyFM_Error) {
|
|
146
|
+
throw error;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
throw new DyFM_Error({
|
|
150
|
+
errorCode: 'DyNTS-SQLITE-READ-003',
|
|
151
|
+
message: `Opening/reading the SQLite DB failed: '${dbPath}'.\n error: `
|
|
144
152
|
+ `${DyFM_Error.getErrorMessage(error)}`,
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
153
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
154
|
+
error: error,
|
|
155
|
+
});
|
|
156
|
+
} finally {
|
|
157
|
+
if (db) {
|
|
158
|
+
db.close();
|
|
159
|
+
}
|
|
153
160
|
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** A SELECT-only guard: nem-SELECT/WITH kezdetű VAGY multi-statement SQL → strukturált hiba. */
|
|
164
|
+
private static assertSelectOnly(sql: string): void {
|
|
165
|
+
const trimmed: string = (sql ?? '').trim();
|
|
154
166
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
message: 'Only read-only SELECT/WITH queries are allowed (the reader is read-only).',
|
|
162
|
-
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
// Multi-statement (`;` után további nem-üres tartalom) tiltott — egy SELECT mögé nem rejthető írás.
|
|
166
|
-
const withoutTrailingSemicolon: string = trimmed.replace(/;\s*$/, '');
|
|
167
|
-
if (withoutTrailingSemicolon.includes(';')) {
|
|
168
|
-
throw new DyFM_Error({
|
|
169
|
-
errorCode: 'DyNTS-SQLITE-READONLY-001',
|
|
170
|
-
message: 'Multi-statement SQL is not allowed (only a single read-only SELECT/WITH).',
|
|
171
|
-
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
172
|
-
});
|
|
173
|
-
}
|
|
167
|
+
if (!trimmed.length || !DyNTS_Sqlite_Reader_Util.SELECT_ONLY.test(trimmed)) {
|
|
168
|
+
throw new DyFM_Error({
|
|
169
|
+
errorCode: 'DyNTS-SQLITE-READONLY-001',
|
|
170
|
+
message: 'Only read-only SELECT/WITH queries are allowed (the reader is read-only).',
|
|
171
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
172
|
+
});
|
|
174
173
|
}
|
|
174
|
+
// Multi-statement (`;` után további nem-üres tartalom) tiltott — egy SELECT mögé nem rejthető írás.
|
|
175
|
+
const withoutTrailingSemicolon: string = trimmed.replace(/;\s*$/, '');
|
|
175
176
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
177
|
+
if (withoutTrailingSemicolon.includes(';')) {
|
|
178
|
+
throw new DyFM_Error({
|
|
179
|
+
errorCode: 'DyNTS-SQLITE-READONLY-001',
|
|
180
|
+
message: 'Multi-statement SQL is not allowed (only a single read-only SELECT/WITH).',
|
|
181
|
+
issuer: DyNTS_Sqlite_Reader_Util.issuer,
|
|
182
|
+
});
|
|
181
183
|
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Egy tábla létezésének read-only ellenőrzése (`sqlite_master`). */
|
|
187
|
+
private static tableExists(db: Database.Database, table: string): boolean {
|
|
188
|
+
const found: unknown =
|
|
189
|
+
db.prepare('SELECT name FROM sqlite_master WHERE type=\'table\' AND name=?').get(table);
|
|
190
|
+
|
|
191
|
+
return Boolean(found);
|
|
192
|
+
}
|
|
182
193
|
}
|
|
183
194
|
|
|
184
195
|
/** A `PRAGMA table_info` nyers sor-shape-je (a `better-sqlite3` ezt adja vissza). */
|
package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { DyFM_DataModel_Params, DyFM_Metadata, DyFM_Object } from '@futdevpro/fsm-dynamo';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
* `
|
|
5
|
+
* `DyNTS_LVS_VectorPersist` (BFR-AM-001) — egy MongoDB-ben perzistált vektor-rekord, ami a
|
|
6
6
|
* `LVS_VectorPool_ControlService` **in-memory** pool-jának tartós háttértára. A FAM (fdp-agent-memory)
|
|
7
7
|
* workaround-ját generalizálja: a vektorok a SAJÁT Mongo-ban élnek, boot-kor a memória-pool-ba
|
|
8
8
|
* hidratálódnak — **NEM** MongoDB Atlas Vector Search.
|
|
@@ -15,7 +15,7 @@ import { DyFM_DataModel_Params, DyFM_Metadata, DyFM_Object } from '@futdevpro/fs
|
|
|
15
15
|
* ősből jön. A `metadata` `unknown` (Mongoose `Mixed`) — a fogyasztó tetszőleges kísérő-adatot tárolhat
|
|
16
16
|
* (a séma NEM validál); az `embedding` `number[]` a perzistált nyers vektor.
|
|
17
17
|
*/
|
|
18
|
-
export class
|
|
18
|
+
export class DyNTS_LVS_VectorPersist extends DyFM_Metadata {
|
|
19
19
|
|
|
20
20
|
/** A pool-on belüli vektor-kulcs (a `pool.addVector(vectorId, ...)` ezzel illeszt). */
|
|
21
21
|
vectorId?: string;
|
|
@@ -32,8 +32,9 @@ export class DyNTS_LVS_VectorPersist_DataModel extends DyFM_Metadata {
|
|
|
32
32
|
/** Opcionális fogyasztó-specifikus kísérő-adat (Mongoose `Mixed`; a séma NEM validál). */
|
|
33
33
|
metadata?: unknown;
|
|
34
34
|
|
|
35
|
-
constructor(set?: Partial<
|
|
35
|
+
constructor(set?: Partial<DyNTS_LVS_VectorPersist>) {
|
|
36
36
|
super(set);
|
|
37
|
+
|
|
37
38
|
if (set) {
|
|
38
39
|
DyFM_Object.cleanAssign(this, set);
|
|
39
40
|
}
|
|
@@ -46,8 +47,8 @@ export class DyNTS_LVS_VectorPersist_DataModel extends DyFM_Metadata {
|
|
|
46
47
|
* `vectorId` + `collectionKey` index-elt (a hidratálás `collectionKey`-re, az upsert/remove a
|
|
47
48
|
* `vectorId`+`collectionKey` párra szűr).
|
|
48
49
|
*/
|
|
49
|
-
export const
|
|
50
|
-
new DyFM_DataModel_Params<
|
|
50
|
+
export const DyNTS_lvsVectorPersist_dataParams: DyFM_DataModel_Params<DyNTS_LVS_VectorPersist> =
|
|
51
|
+
new DyFM_DataModel_Params<DyNTS_LVS_VectorPersist>({
|
|
51
52
|
dataName: 'dynts_lvs_vector',
|
|
52
53
|
properties: {
|
|
53
54
|
vectorId: { type: 'string', index: true, required: true },
|
|
@@ -5,12 +5,12 @@ import { DyNTS_global_settings } from '../../../_collections/global-settings.con
|
|
|
5
5
|
import { DyNTS_GlobalService } from '../../../_services/core/global.service';
|
|
6
6
|
import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
|
|
7
7
|
import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
|
|
8
|
-
import {
|
|
8
|
+
import { DyNTS_LVS_VectorPersist } from '../_models/data-models/lvs-vector-persist.data-model';
|
|
9
9
|
import { DyNTS_LVS_VectorPersist_DataService } from './lvs-vector-persist.data-service';
|
|
10
|
-
import {
|
|
10
|
+
import { DyNTS_LVS_PersistentVectorPool_ControlService } from './lvs-persistent-vector-pool.control-service';
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
|
-
* BFR-AM-001 — a `
|
|
13
|
+
* BFR-AM-001 — a `DyNTS_LVS_PersistentVectorPool_ControlService` (kompozíció: in-memory pool + Mongo-persist) spec-jei.
|
|
14
14
|
*
|
|
15
15
|
* A DB-réteget egy IN-MEMORY fake-store helyettesíti: a `DyNTS_LVS_VectorPersist_DataService.prototype`
|
|
16
16
|
* `upsertVector`/`removeVector`/`findByCollectionKey` metódusait stub-oljuk (a `getDBService` eager
|
|
@@ -18,12 +18,12 @@ import { DyNTS_LVS_PersistentVectorPool } from './lvs-persistent-vector-pool.con
|
|
|
18
18
|
* így a perzisztencia memóriában modellezett, ÉLŐ Mongo nélkül (skip-guard nem szükséges). A fő bizonyíték:
|
|
19
19
|
* **persist-then-hydrate round-trip** újraépíti a VALÓDI `LVS_VectorPool_ControlService` in-memory indexét.
|
|
20
20
|
*/
|
|
21
|
-
describe('|
|
|
21
|
+
describe('| DyNTS_LVS_PersistentVectorPool_ControlService', () => {
|
|
22
22
|
|
|
23
23
|
let mockDBService: jasmine.SpyObj<{ find: () => Promise<unknown[]>; findOne: () => Promise<unknown> }>;
|
|
24
24
|
|
|
25
25
|
/** Fake tartós tár: `collectionKey` → (`vectorId` → rekord). A persist-stubok ezt írják/olvassák. */
|
|
26
|
-
let store: Map<string, Map<string,
|
|
26
|
+
let store: Map<string, Map<string, DyNTS_LVS_VectorPersist>>;
|
|
27
27
|
|
|
28
28
|
beforeAll(() => {
|
|
29
29
|
if (!DyNTS_global_settings.systemShortCodeName) {
|
|
@@ -44,16 +44,16 @@ describe('| DyNTS_LVS_PersistentVectorPool', () => {
|
|
|
44
44
|
spyOn(DyNTS_GlobalService, 'getDBService').and.returnValue(mockDBService as never);
|
|
45
45
|
|
|
46
46
|
// A fake tartós tár + a persist-service prototype-stubok (memóriában modellezett Mongo).
|
|
47
|
-
store = new Map<string, Map<string,
|
|
47
|
+
store = new Map<string, Map<string, DyNTS_LVS_VectorPersist>>();
|
|
48
48
|
|
|
49
49
|
spyOn(DyNTS_LVS_VectorPersist_DataService.prototype, 'upsertVector').and.callFake(
|
|
50
50
|
(set: { collectionKey: string; vectorId: string; embedding: number[]; metadata?: unknown }) => {
|
|
51
|
-
let coll: Map<string,
|
|
51
|
+
let coll: Map<string, DyNTS_LVS_VectorPersist> | undefined = store.get(set.collectionKey);
|
|
52
52
|
if (!coll) {
|
|
53
|
-
coll = new Map<string,
|
|
53
|
+
coll = new Map<string, DyNTS_LVS_VectorPersist>();
|
|
54
54
|
store.set(set.collectionKey, coll);
|
|
55
55
|
}
|
|
56
|
-
coll.set(set.vectorId, new
|
|
56
|
+
coll.set(set.vectorId, new DyNTS_LVS_VectorPersist({
|
|
57
57
|
collectionKey: set.collectionKey,
|
|
58
58
|
vectorId: set.vectorId,
|
|
59
59
|
embedding: set.embedding,
|
|
@@ -85,12 +85,12 @@ describe('| DyNTS_LVS_PersistentVectorPool', () => {
|
|
|
85
85
|
};
|
|
86
86
|
|
|
87
87
|
it('| should persist on addVector and store the dimension', async () => {
|
|
88
|
-
const pool:
|
|
89
|
-
new
|
|
88
|
+
const pool: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
89
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
90
90
|
|
|
91
91
|
await pool.addVector('v1', [0.1, 0.2, 0.3], { tag: 'x' });
|
|
92
92
|
|
|
93
|
-
const persisted:
|
|
93
|
+
const persisted: DyNTS_LVS_VectorPersist | undefined = store.get('pool-a')?.get('v1');
|
|
94
94
|
expect(persisted).toBeDefined();
|
|
95
95
|
expect(persisted.embedding).toEqual([0.1, 0.2, 0.3]);
|
|
96
96
|
expect(persisted.dimensions).toBe(3);
|
|
@@ -101,16 +101,16 @@ describe('| DyNTS_LVS_PersistentVectorPool', () => {
|
|
|
101
101
|
|
|
102
102
|
it('| persist-then-hydrate round-trip rebuilds the in-memory pool', async () => {
|
|
103
103
|
// (1) Egy pool-ba 3 vektort persistálunk.
|
|
104
|
-
const writer:
|
|
105
|
-
new
|
|
104
|
+
const writer: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
105
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
106
106
|
await writer.addVector('v0', axis(4, 0));
|
|
107
107
|
await writer.addVector('v1', axis(4, 1));
|
|
108
108
|
await writer.addVector('v2', axis(4, 2));
|
|
109
109
|
expect(store.get('pool-a').size).toBe(3);
|
|
110
110
|
|
|
111
111
|
// (2) Egy FRISS wrapper (üres memória-pool) — szerver-restart szimuláció.
|
|
112
|
-
const rebuilt:
|
|
113
|
-
new
|
|
112
|
+
const rebuilt: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
113
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
114
114
|
expect(rebuilt.getPool().getAll().size).toBe(0);
|
|
115
115
|
|
|
116
116
|
// (3) Hidratálás Mongo-ból → a memória-pool újraépül.
|
|
@@ -126,38 +126,38 @@ describe('| DyNTS_LVS_PersistentVectorPool', () => {
|
|
|
126
126
|
});
|
|
127
127
|
|
|
128
128
|
it('| should only hydrate the matching collectionKey (pool partitioning)', async () => {
|
|
129
|
-
const a:
|
|
130
|
-
new
|
|
131
|
-
const b:
|
|
132
|
-
new
|
|
129
|
+
const a: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
130
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
131
|
+
const b: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
132
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-b', issuer: 'test' });
|
|
133
133
|
|
|
134
134
|
await a.addVector('a1', axis(3, 0));
|
|
135
135
|
await b.addVector('b1', axis(3, 1));
|
|
136
136
|
await b.addVector('b2', axis(3, 2));
|
|
137
137
|
|
|
138
|
-
const rebuiltA:
|
|
139
|
-
new
|
|
138
|
+
const rebuiltA: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
139
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
140
140
|
expect(await rebuiltA.hydrateFromMongo()).toBe(1);
|
|
141
141
|
expect(Array.from(rebuiltA.getPool().getAll().keys())).toEqual(['a1']);
|
|
142
142
|
});
|
|
143
143
|
|
|
144
144
|
it('| upsert (re-add) is reflected in both the pool and the store', async () => {
|
|
145
|
-
const pool:
|
|
146
|
-
new
|
|
145
|
+
const pool: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
146
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
147
147
|
|
|
148
148
|
await pool.addVector('v1', [1, 0, 0]);
|
|
149
149
|
await pool.updateVector('v1', [0, 1, 0], { v: 2 });
|
|
150
150
|
|
|
151
151
|
// a tárban EGY rekord van, a frissített értékkel + dimenzióval:
|
|
152
152
|
expect(store.get('pool-a').size).toBe(1);
|
|
153
|
-
const persisted:
|
|
153
|
+
const persisted: DyNTS_LVS_VectorPersist = store.get('pool-a').get('v1');
|
|
154
154
|
expect(persisted.embedding).toEqual([0, 1, 0]);
|
|
155
155
|
expect(persisted.dimensions).toBe(3);
|
|
156
156
|
expect(persisted.metadata).toEqual({ v: 2 });
|
|
157
157
|
|
|
158
158
|
// és egy friss hidratálás a frissített vektort hozza:
|
|
159
|
-
const rebuilt:
|
|
160
|
-
new
|
|
159
|
+
const rebuilt: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
160
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
161
161
|
await rebuilt.hydrateFromMongo();
|
|
162
162
|
const hits: LVS_SearchResult[] = rebuilt.search([0, 1, 0], 1, LVS_Search_Mode.cosineSimilarity);
|
|
163
163
|
expect(hits[0].id).toBe('v1');
|
|
@@ -165,8 +165,8 @@ describe('| DyNTS_LVS_PersistentVectorPool', () => {
|
|
|
165
165
|
});
|
|
166
166
|
|
|
167
167
|
it('| removeVector is reflected in both the pool and the store', async () => {
|
|
168
|
-
const pool:
|
|
169
|
-
new
|
|
168
|
+
const pool: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
169
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
170
170
|
|
|
171
171
|
await pool.addVector('v1', axis(3, 0));
|
|
172
172
|
await pool.addVector('v2', axis(3, 1));
|
|
@@ -181,15 +181,15 @@ describe('| DyNTS_LVS_PersistentVectorPool', () => {
|
|
|
181
181
|
});
|
|
182
182
|
|
|
183
183
|
it('| hydrate(records) rebuilds the pool without a Mongo read and skips incomplete records', () => {
|
|
184
|
-
const pool:
|
|
185
|
-
new
|
|
184
|
+
const pool: DyNTS_LVS_PersistentVectorPool_ControlService =
|
|
185
|
+
new DyNTS_LVS_PersistentVectorPool_ControlService({ collectionKey: 'pool-a', issuer: 'test' });
|
|
186
186
|
|
|
187
187
|
const loaded: number = pool.hydrate([
|
|
188
|
-
new
|
|
189
|
-
new
|
|
188
|
+
new DyNTS_LVS_VectorPersist({ collectionKey: 'pool-a', vectorId: 'v1', embedding: axis(2, 0) }),
|
|
189
|
+
new DyNTS_LVS_VectorPersist({ collectionKey: 'pool-a', vectorId: 'v2', embedding: axis(2, 1) }),
|
|
190
190
|
// hiányos rekordok (átugorva):
|
|
191
|
-
new
|
|
192
|
-
new
|
|
191
|
+
new DyNTS_LVS_VectorPersist({ collectionKey: 'pool-a', vectorId: 'v3', embedding: [] }),
|
|
192
|
+
new DyNTS_LVS_VectorPersist({ collectionKey: 'pool-a', embedding: axis(2, 0) }),
|
|
193
193
|
]);
|
|
194
194
|
|
|
195
195
|
expect(loaded).toBe(2);
|
package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts
CHANGED
|
@@ -3,12 +3,12 @@ import { DyFM_Log } from '@futdevpro/fsm-dynamo';
|
|
|
3
3
|
|
|
4
4
|
import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
|
|
5
5
|
import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
|
|
6
|
-
import {
|
|
6
|
+
import { DyNTS_LVS_VectorPersist } from '../_models/data-models/lvs-vector-persist.data-model';
|
|
7
7
|
import { DyNTS_LVS_VectorPersist_DataService } from './lvs-vector-persist.data-service';
|
|
8
8
|
import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* `
|
|
11
|
+
* `DyNTS_LVS_PersistentVectorPool_ControlService` (BFR-AM-001) — egy `LVS_VectorPool_ControlService` in-memory pool +
|
|
12
12
|
* MongoDB-perzisztencia **kompozíciója**. A FAM (fdp-agent-memory) `FAM_VectorSearch_ControlService`
|
|
13
13
|
* persist+hydrate workaround-ját generalizálja bedrock-szintre: a vektorok a SAJÁT Mongo-ban
|
|
14
14
|
* (`dynts_lvs_vector`) élnek, boot-kor a memória-pool-ba hidratálódnak — **NEM** MongoDB Atlas
|
|
@@ -24,7 +24,7 @@ import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service
|
|
|
24
24
|
* `getDBService`-t hív, ezért a wrapper NEM tart élő data-service-mezőt — minden DB-művelet előtt lazy
|
|
25
25
|
* `new DyNTS_LVS_VectorPersist_DataService(...)` (a `getPersistService` ezt adja).
|
|
26
26
|
*/
|
|
27
|
-
export class
|
|
27
|
+
export class DyNTS_LVS_PersistentVectorPool_ControlService {
|
|
28
28
|
|
|
29
29
|
/** A wrapped in-memory pool (a Dynamo LVS engine; VÁLTOZATLAN — kompozíció, nem módosítás). */
|
|
30
30
|
private readonly pool: LVS_VectorPool_ControlService;
|
|
@@ -98,12 +98,13 @@ export class DyNTS_LVS_PersistentVectorPool {
|
|
|
98
98
|
* Visszaadja a betöltött vektorok számát. **NEM** Atlas — saját Mongo + in-memory pool.
|
|
99
99
|
*/
|
|
100
100
|
async hydrateFromMongo(): Promise<number> {
|
|
101
|
-
const records:
|
|
101
|
+
const records: DyNTS_LVS_VectorPersist[] =
|
|
102
102
|
await this.getPersistService().findByCollectionKey(this.collectionKey);
|
|
103
103
|
|
|
104
104
|
this.pool.clearPool();
|
|
105
105
|
|
|
106
106
|
let loaded: number = 0;
|
|
107
|
+
|
|
107
108
|
for (const record of records) {
|
|
108
109
|
if (!record.vectorId || !record.embedding?.length) {
|
|
109
110
|
continue;
|
|
@@ -115,6 +116,7 @@ export class DyNTS_LVS_PersistentVectorPool {
|
|
|
115
116
|
DyFM_Log.log(
|
|
116
117
|
`[DyNTS LVS hydrate] '${this.collectionKey}' pool: ${loaded} vektor betöltve az in-memory pool-ba.`,
|
|
117
118
|
);
|
|
119
|
+
|
|
118
120
|
return loaded;
|
|
119
121
|
}
|
|
120
122
|
|
|
@@ -123,9 +125,10 @@ export class DyNTS_LVS_PersistentVectorPool {
|
|
|
123
125
|
* kezében van a rekord-halmaz (pl. batch-boot). A pool-t előbb üríti (idempotens). Visszaadja a
|
|
124
126
|
* betöltött vektorok számát.
|
|
125
127
|
*/
|
|
126
|
-
hydrate(records:
|
|
128
|
+
hydrate(records: DyNTS_LVS_VectorPersist[]): number {
|
|
127
129
|
this.pool.clearPool();
|
|
128
130
|
let loaded: number = 0;
|
|
131
|
+
|
|
129
132
|
for (const record of records) {
|
|
130
133
|
if (!record.vectorId || !record.embedding?.length) {
|
|
131
134
|
continue;
|
|
@@ -133,6 +136,7 @@ export class DyNTS_LVS_PersistentVectorPool {
|
|
|
133
136
|
this.pool.addVector(record.vectorId, record.embedding);
|
|
134
137
|
loaded++;
|
|
135
138
|
}
|
|
139
|
+
|
|
136
140
|
return loaded;
|
|
137
141
|
}
|
|
138
142
|
|