@futdevpro/nts-dynamo 1.15.58 → 1.15.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dynamo/logs/cicd-pipeline/output.log +1746 -1819
- package/.dynamo/logs/cicd-pipeline/status.json +37 -37
- package/.github/workflows/main.yml +432 -426
- package/build/_collections/global-settings.const.d.ts.map +1 -1
- package/build/_collections/global-settings.const.js +6 -0
- package/build/_collections/global-settings.const.js.map +1 -1
- package/build/_collections/mongo-reconnect-guard.util.d.ts +74 -0
- package/build/_collections/mongo-reconnect-guard.util.d.ts.map +1 -0
- package/build/_collections/mongo-reconnect-guard.util.js +111 -0
- package/build/_collections/mongo-reconnect-guard.util.js.map +1 -0
- package/build/_models/interfaces/global-settings.interface.d.ts +21 -0
- package/build/_models/interfaces/global-settings.interface.d.ts.map +1 -1
- package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.d.ts.map +1 -1
- package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.js +2 -2
- package/build/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.js.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-mock.service.d.ts +1 -1
- package/build/_modules/ai/_services/ai-embedding-mock.service.d.ts.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-mock.service.js.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-provider.registry.d.ts.map +1 -1
- package/build/_modules/ai/_services/ai-embedding-provider.registry.js.map +1 -1
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.d.ts +1 -1
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.d.ts.map +1 -1
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.js +3 -3
- package/build/_modules/ai/_services/lmstudio-embedding.control-service.js.map +1 -1
- package/build/_modules/ai/index.d.ts +2 -0
- package/build/_modules/ai/index.d.ts.map +1 -1
- package/build/_modules/ai/index.js +4 -0
- package/build/_modules/ai/index.js.map +1 -1
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.d.ts +17 -17
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.d.ts.map +1 -1
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.js +21 -21
- package/build/_modules/data-readers/_collections/dynts-sqlite-reader.util.js.map +1 -1
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.d.ts +4 -4
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.d.ts.map +1 -1
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.js +5 -5
- package/build/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.js.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.d.ts +4 -4
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.d.ts.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.js +4 -4
- package/build/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.js.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.d.ts +6 -6
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.d.ts.map +1 -1
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.js +5 -5
- package/build/_modules/local-vector-search/_services/lvs-vector-persist.data-service.js.map +1 -1
- package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.d.ts +1 -1
- package/build/_modules/mcp/_models/interfaces/dynts-mcp.interface.js +1 -1
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.d.ts +4 -4
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.d.ts.map +1 -1
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.js +6 -6
- package/build/_modules/mcp/_services/dynts-mcp-server.service-base.js.map +1 -1
- package/build/_modules/mcp/_services/dynts-mcp.adapter.d.ts +11 -11
- package/build/_modules/mcp/_services/dynts-mcp.adapter.d.ts.map +1 -1
- package/build/_modules/mcp/_services/dynts-mcp.adapter.js +16 -11
- package/build/_modules/mcp/_services/dynts-mcp.adapter.js.map +1 -1
- package/build/_modules/mcp/index.js +1 -1
- package/build/_modules/mcp/index.js.map +1 -1
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.d.ts +3 -3
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.d.ts.map +1 -1
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.js +4 -4
- package/build/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.js.map +1 -1
- package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.d.ts.map +1 -1
- package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.js +9 -0
- package/build/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.js.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.d.ts +3 -3
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.d.ts.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.js +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.control-service.js.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.d.ts +7 -7
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.d.ts.map +1 -1
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.js +2 -2
- package/build/_modules/scoped-config/_services/dynts-scoped-config.data-service.js.map +1 -1
- package/build/_services/core/global.service.d.ts.map +1 -1
- package/build/_services/core/global.service.js +15 -2
- package/build/_services/core/global.service.js.map +1 -1
- package/build/_services/server/app.server.d.ts.map +1 -1
- package/build/_services/server/app.server.js +21 -0
- package/build/_services/server/app.server.js.map +1 -1
- package/package.json +1 -1
- package/src/_collections/global-settings.const.ts +7 -0
- package/src/_collections/mongo-reconnect-guard.util.spec.ts +52 -0
- package/src/_collections/mongo-reconnect-guard.util.ts +172 -0
- package/src/_models/interfaces/global-settings.interface.ts +22 -0
- package/src/_modules/ai/_modules/document-ai/_collections/dai-code-chunking.util.ts +39 -7
- package/src/_modules/ai/_services/ai-embedding-mock.service.ts +18 -4
- package/src/_modules/ai/_services/ai-embedding-provider.registry.ts +4 -0
- package/src/_modules/ai/_services/lmstudio-embedding.control-service.ts +26 -5
- package/src/_modules/ai/index.ts +5 -0
- package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.spec.ts +145 -130
- package/src/_modules/data-readers/_collections/dynts-sqlite-reader.util.ts +131 -120
- package/src/_modules/local-vector-search/_models/data-models/lvs-vector-persist.data-model.ts +6 -5
- package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.spec.ts +35 -35
- package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts +9 -5
- package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts +11 -11
- package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts +19 -17
- package/src/_modules/mcp/_models/interfaces/dynts-mcp.interface.ts +1 -1
- package/src/_modules/mcp/_services/dynts-mcp-server.service-base.spec.ts +123 -114
- package/src/_modules/mcp/_services/dynts-mcp-server.service-base.ts +44 -39
- package/src/_modules/mcp/_services/dynts-mcp.adapter.ts +114 -103
- package/src/_modules/mcp/index.ts +1 -1
- package/src/_modules/scoped-config/_models/data-models/dynts-scoped-config.data-model.ts +5 -4
- package/src/_modules/scoped-config/_models/interfaces/dynts-scoped-config.interface.ts +0 -2
- package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.spec.ts +19 -13
- package/src/_modules/scoped-config/_services/dynts-scoped-config.control-service.ts +37 -21
- package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.spec.ts +11 -6
- package/src/_modules/scoped-config/_services/dynts-scoped-config.data-service.ts +17 -14
- package/src/_services/core/global.service.spec.ts +17 -0
- package/src/_services/core/global.service.ts +19 -5
- package/src/_services/server/app.server.ts +22 -1
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import * as ts from 'typescript';
|
|
2
|
+
|
|
2
3
|
import { DyFM_Log } from '@futdevpro/fsm-dynamo';
|
|
4
|
+
|
|
3
5
|
import { DyNTS_DAI_CodeChunk, DyNTS_DAI_CodeChunkType } from '../_models/interfaces/dai-code-chunk.interface';
|
|
4
6
|
|
|
5
7
|
/** A kód-chunker méret-paraméterei (a FAM/CCAP konstansokkal egyező default-ok). */
|
|
@@ -80,13 +82,14 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
80
82
|
* @returns A szemantikus kód-chunkok listája (0-alapú `chunkIndex`-szel).
|
|
81
83
|
*/
|
|
82
84
|
static chunkTs(code: string, opts?: DyNTS_DAI_CodeChunking_Options): DyNTS_DAI_CodeChunk[] {
|
|
83
|
-
if (!code
|
|
85
|
+
if (!code?.trim()) {
|
|
84
86
|
return [];
|
|
85
87
|
}
|
|
86
88
|
|
|
87
89
|
const sizing: DyNTS_DAI_CodeChunking_Sizing = this.resolveSizing(opts);
|
|
88
90
|
|
|
89
91
|
let blocks: DyNTS_DAI_CodeBlock[] = [];
|
|
92
|
+
|
|
90
93
|
try {
|
|
91
94
|
const sourceFile: ts.SourceFile = ts.createSourceFile(
|
|
92
95
|
'chunk-input.ts',
|
|
@@ -95,6 +98,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
95
98
|
/* setParentNodes */ true,
|
|
96
99
|
ts.ScriptKind.TSX
|
|
97
100
|
);
|
|
101
|
+
|
|
98
102
|
blocks = this.identifyBlocks(sourceFile, code, sizing);
|
|
99
103
|
} catch (error) {
|
|
100
104
|
// Parse-hiba (degenerált forrás) → a tiny-fallback veszi át lentebb.
|
|
@@ -114,6 +118,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
114
118
|
const lineStartOffsets: number[] = this.computeLineStartOffsets(code);
|
|
115
119
|
|
|
116
120
|
const chunks: DyNTS_DAI_CodeChunk[] = [];
|
|
121
|
+
|
|
117
122
|
for (const block of blocks) {
|
|
118
123
|
// Zaj-szűrés: CSAK a névtelen `generic` blokkokat dobjuk a minChunkSize alatt. A nevesített
|
|
119
124
|
// szemantikus deklarációk (type-alias / rövid function / class-fej / metódus / interface /
|
|
@@ -156,11 +161,13 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
156
161
|
if (this.isTsLike(filename)) {
|
|
157
162
|
return this.chunkTs(content, opts);
|
|
158
163
|
}
|
|
164
|
+
|
|
159
165
|
// Nem-kód (pl. .md / .txt / ismeretlen): nyers fallback chunkok — a tartalmat megőrizzük,
|
|
160
166
|
// de NEM próbáljuk szemantikusan darabolni (a markdown-darabolás külön util-é).
|
|
161
|
-
if (!content
|
|
167
|
+
if (!content?.trim()) {
|
|
162
168
|
return [];
|
|
163
169
|
}
|
|
170
|
+
|
|
164
171
|
return this.buildFallbackChunks(content, this.resolveSizing(opts));
|
|
165
172
|
}
|
|
166
173
|
|
|
@@ -174,6 +181,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
174
181
|
const maxChunkSize: number = Math.max(1, opts?.maxChunkSize ?? this.DEFAULT_MAX_CHUNK_SIZE);
|
|
175
182
|
const overlap: number = Math.max(0, Math.min(opts?.overlap ?? this.DEFAULT_OVERLAP, maxChunkSize - 1));
|
|
176
183
|
const minChunkSize: number = Math.max(0, opts?.minChunkSize ?? this.DEFAULT_MIN_CHUNK_SIZE);
|
|
184
|
+
|
|
177
185
|
return { maxChunkSize: maxChunkSize, overlap: overlap, minChunkSize: minChunkSize };
|
|
178
186
|
}
|
|
179
187
|
|
|
@@ -198,6 +206,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
198
206
|
return;
|
|
199
207
|
}
|
|
200
208
|
const text: string = code.substring(importStart, importEnd);
|
|
209
|
+
|
|
201
210
|
if (text.trim().length) {
|
|
202
211
|
blocks.push({
|
|
203
212
|
text: text,
|
|
@@ -215,6 +224,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
215
224
|
for (const statement of sourceFile.statements) {
|
|
216
225
|
if (this.isImportLike(statement)) {
|
|
217
226
|
const start: number = statement.getStart(sourceFile, /* includeJsDocComment */ true);
|
|
227
|
+
|
|
218
228
|
if (importStart < 0) {
|
|
219
229
|
importStart = start;
|
|
220
230
|
}
|
|
@@ -229,7 +239,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
229
239
|
const start: number = statement.getStart(sourceFile, /* includeJsDocComment */ true);
|
|
230
240
|
const end: number = statement.getEnd();
|
|
231
241
|
const text: string = code.substring(start, end);
|
|
232
|
-
const symbolSegments: string[] = detected.name ? [`${detected.chunkType}:${detected.name}`] : [];
|
|
242
|
+
const symbolSegments: string[] = detected.name ? [ `${detected.chunkType}:${detected.name}` ] : [];
|
|
233
243
|
|
|
234
244
|
// Nagy osztály → metódusonkénti bontás (osztály-fej + metódusok).
|
|
235
245
|
if (ts.isClassDeclaration(statement) && text.length > sizing.maxChunkSize) {
|
|
@@ -258,10 +268,12 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
258
268
|
if (ts.isImportDeclaration(statement) || ts.isImportEqualsDeclaration(statement)) {
|
|
259
269
|
return true;
|
|
260
270
|
}
|
|
271
|
+
|
|
261
272
|
// `export { ... } from '...'` / `export * from '...'` — csak ha van moduleSpecifier.
|
|
262
273
|
if (ts.isExportDeclaration(statement) && !!statement.moduleSpecifier) {
|
|
263
274
|
return true;
|
|
264
275
|
}
|
|
276
|
+
|
|
265
277
|
return false;
|
|
266
278
|
}
|
|
267
279
|
|
|
@@ -270,18 +282,23 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
270
282
|
if (ts.isClassDeclaration(statement)) {
|
|
271
283
|
return { chunkType: 'class', name: statement.name?.getText() };
|
|
272
284
|
}
|
|
285
|
+
|
|
273
286
|
if (ts.isInterfaceDeclaration(statement)) {
|
|
274
287
|
return { chunkType: 'interface', name: statement.name.getText() };
|
|
275
288
|
}
|
|
289
|
+
|
|
276
290
|
if (ts.isEnumDeclaration(statement)) {
|
|
277
291
|
return { chunkType: 'enum', name: statement.name.getText() };
|
|
278
292
|
}
|
|
293
|
+
|
|
279
294
|
if (ts.isTypeAliasDeclaration(statement)) {
|
|
280
295
|
return { chunkType: 'type', name: statement.name.getText() };
|
|
281
296
|
}
|
|
297
|
+
|
|
282
298
|
if (ts.isFunctionDeclaration(statement)) {
|
|
283
299
|
return { chunkType: 'function', name: statement.name?.getText() };
|
|
284
300
|
}
|
|
301
|
+
|
|
285
302
|
if (ts.isVariableStatement(statement)) {
|
|
286
303
|
const firstDecl: ts.VariableDeclaration | undefined = statement.declarationList.declarations[0];
|
|
287
304
|
const name: string | undefined = firstDecl && ts.isIdentifier(firstDecl.name)
|
|
@@ -291,8 +308,10 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
291
308
|
const initializer: ts.Expression | undefined = firstDecl?.initializer;
|
|
292
309
|
const isFn: boolean = !!initializer
|
|
293
310
|
&& (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer));
|
|
311
|
+
|
|
294
312
|
return { chunkType: isFn ? 'function' : 'variable', name: name };
|
|
295
313
|
}
|
|
314
|
+
|
|
296
315
|
return { chunkType: 'generic', name: undefined };
|
|
297
316
|
}
|
|
298
317
|
|
|
@@ -311,18 +330,19 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
311
330
|
const classSegment: string = `class:${className ?? 'anonymous'}`;
|
|
312
331
|
const classStart: number = classDecl.getStart(sourceFile, /* includeJsDocComment */ true);
|
|
313
332
|
|
|
314
|
-
const members: ts.ClassElement[] = [...classDecl.members];
|
|
333
|
+
const members: ts.ClassElement[] = [ ...classDecl.members ];
|
|
315
334
|
|
|
316
335
|
// Osztály-fej: a class kezdetétől az első tag kezdetéig (dekorátorok/JSDoc + `class X {`-rész).
|
|
317
336
|
const headerEnd: number = members.length
|
|
318
337
|
? members[0].getStart(sourceFile, /* includeJsDocComment */ true)
|
|
319
338
|
: classDecl.getEnd();
|
|
320
339
|
const headerText: string = code.substring(classStart, headerEnd);
|
|
340
|
+
|
|
321
341
|
if (headerText.trim().length) {
|
|
322
342
|
blocks.push({
|
|
323
343
|
text: headerText,
|
|
324
344
|
chunkType: 'class',
|
|
325
|
-
symbolSegments: [classSegment],
|
|
345
|
+
symbolSegments: [ classSegment ],
|
|
326
346
|
symbolName: className,
|
|
327
347
|
charStart: classStart,
|
|
328
348
|
charEnd: headerEnd,
|
|
@@ -334,6 +354,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
334
354
|
const memberStart: number = member.getStart(sourceFile, /* includeJsDocComment */ true);
|
|
335
355
|
const memberEnd: number = member.getEnd();
|
|
336
356
|
const memberText: string = code.substring(memberStart, memberEnd);
|
|
357
|
+
|
|
337
358
|
if (!memberText.trim().length) {
|
|
338
359
|
continue;
|
|
339
360
|
}
|
|
@@ -349,7 +370,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
349
370
|
blocks.push({
|
|
350
371
|
text: memberText,
|
|
351
372
|
chunkType: memberKind,
|
|
352
|
-
symbolSegments: [classSegment, memberSegment],
|
|
373
|
+
symbolSegments: [ classSegment, memberSegment ],
|
|
353
374
|
symbolName: memberName,
|
|
354
375
|
charStart: memberStart,
|
|
355
376
|
charEnd: memberEnd,
|
|
@@ -367,6 +388,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
367
388
|
) {
|
|
368
389
|
return 'method';
|
|
369
390
|
}
|
|
391
|
+
|
|
370
392
|
return 'variable';
|
|
371
393
|
}
|
|
372
394
|
|
|
@@ -378,6 +400,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
378
400
|
): DyNTS_DAI_CodeChunk {
|
|
379
401
|
const lineStart: number = this.lineOfOffset(block.charStart, lineStartOffsets);
|
|
380
402
|
const lineEnd: number = this.lineOfOffset(Math.max(block.charStart, block.charEnd - 1), lineStartOffsets);
|
|
403
|
+
|
|
381
404
|
return {
|
|
382
405
|
content: block.text,
|
|
383
406
|
chunkType: block.chunkType,
|
|
@@ -448,6 +471,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
448
471
|
*/
|
|
449
472
|
private static buildFallbackChunks(content: string, sizing: DyNTS_DAI_CodeChunking_Sizing): DyNTS_DAI_CodeChunk[] {
|
|
450
473
|
const trimmed: string = content.trim();
|
|
474
|
+
|
|
451
475
|
if (!trimmed) {
|
|
452
476
|
return [];
|
|
453
477
|
}
|
|
@@ -457,6 +481,7 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
457
481
|
// Kis tartalom → egyetlen fallback chunk.
|
|
458
482
|
if (content.length <= sizing.maxChunkSize) {
|
|
459
483
|
const lineEnd: number = this.lineOfOffset(Math.max(0, content.length - 1), lineStartOffsets);
|
|
484
|
+
|
|
460
485
|
return [{
|
|
461
486
|
content: content,
|
|
462
487
|
chunkType: 'fallback',
|
|
@@ -481,7 +506,9 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
481
506
|
charEnd: content.length,
|
|
482
507
|
};
|
|
483
508
|
const out: DyNTS_DAI_CodeChunk[] = [];
|
|
509
|
+
|
|
484
510
|
this.splitLargeBlock(fallbackBlock, sizing, out, lineStartOffsets);
|
|
511
|
+
|
|
485
512
|
return out;
|
|
486
513
|
}
|
|
487
514
|
|
|
@@ -491,12 +518,14 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
491
518
|
* konverziót ad a chunkokhoz.
|
|
492
519
|
*/
|
|
493
520
|
private static computeLineStartOffsets(source: string): number[] {
|
|
494
|
-
const offsets: number[] = [0];
|
|
521
|
+
const offsets: number[] = [ 0 ];
|
|
522
|
+
|
|
495
523
|
for (let i: number = 0; i < source.length; i++) {
|
|
496
524
|
if (source[i] === '\n') {
|
|
497
525
|
offsets.push(i + 1);
|
|
498
526
|
}
|
|
499
527
|
}
|
|
528
|
+
|
|
500
529
|
return offsets;
|
|
501
530
|
}
|
|
502
531
|
|
|
@@ -507,14 +536,17 @@ export class DyNTS_DAI_CodeChunking_Util {
|
|
|
507
536
|
private static lineOfOffset(offset: number, lineStartOffsets: number[]): number {
|
|
508
537
|
let lo: number = 0;
|
|
509
538
|
let hi: number = lineStartOffsets.length - 1;
|
|
539
|
+
|
|
510
540
|
while (lo < hi) {
|
|
511
541
|
const mid: number = (lo + hi + 1) >> 1;
|
|
542
|
+
|
|
512
543
|
if (lineStartOffsets[mid] <= offset) {
|
|
513
544
|
lo = mid;
|
|
514
545
|
} else {
|
|
515
546
|
hi = mid - 1;
|
|
516
547
|
}
|
|
517
548
|
}
|
|
549
|
+
|
|
518
550
|
return lo + 1;
|
|
519
551
|
}
|
|
520
552
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import * as crypto from 'crypto';
|
|
2
|
-
|
|
3
2
|
import { CreateEmbeddingResponse } from 'openai/resources';
|
|
3
|
+
|
|
4
4
|
import { DyFM_AI_Provider, DyFM_AI_ProviderCapabilities, DyFM_AI_Config } from '@futdevpro/fsm-dynamo/ai';
|
|
5
5
|
import { DyFM_DAI_EmbeddingInfo } from '@futdevpro/fsm-dynamo/ai/document-ai';
|
|
6
6
|
|
|
7
|
-
import { DyNTS_AI_Embedding_ServiceBase } from './ai-embedding.service-base';
|
|
8
7
|
import { DyNTS_AI_CostEventCallback } from '../_models/interfaces/dynts-ai-cost-event-callback.interface';
|
|
8
|
+
import { DyNTS_AI_Embedding_ServiceBase } from './ai-embedding.service-base';
|
|
9
9
|
|
|
10
10
|
/** A mock-embedding default dimenziója (kicsi, hogy a teszt-pool gyors legyen, de elég a koszinuszhoz). */
|
|
11
11
|
export const DyNTS_AI_MOCK_EMBEDDING_DIMS: number = 64;
|
|
@@ -61,6 +61,7 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
61
61
|
constructor(set?: DyNTS_AI_Embedding_MockSettings) {
|
|
62
62
|
super();
|
|
63
63
|
this.dims = set?.dims && set.dims > 0 ? set.dims : DyNTS_AI_MOCK_EMBEDDING_DIMS;
|
|
64
|
+
|
|
64
65
|
if (set?.onCostEvent) {
|
|
65
66
|
this.onCostEvent = set.onCostEvent;
|
|
66
67
|
}
|
|
@@ -69,6 +70,7 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
69
70
|
/** A mock-nak nincs külső config-ja; a `setup` a dimenzió override-jára szolgál (config.dimensions). */
|
|
70
71
|
setup(config: DyFM_AI_Config): void {
|
|
71
72
|
const dims: unknown = Reflect.get(config ?? {}, 'dimensions');
|
|
73
|
+
|
|
72
74
|
if (typeof dims === 'number' && dims > 0) {
|
|
73
75
|
this.dims = dims;
|
|
74
76
|
}
|
|
@@ -90,11 +92,12 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
90
92
|
const vector: number[] = this.deterministicVector(`${set.model}::${set.text ?? ''}`);
|
|
91
93
|
const durationMs: number = Date.now() - start;
|
|
92
94
|
|
|
93
|
-
this.emitMockCostEvent('embedding-single', set.model, [set.text ?? ''], durationMs, set.issuer);
|
|
95
|
+
this.emitMockCostEvent('embedding-single', set.model, [ set.text ?? '' ], durationMs, set.issuer);
|
|
94
96
|
|
|
95
97
|
if (set.fullResponse) {
|
|
96
|
-
return this.toResponse([vector], set.model, [set.text ?? '']);
|
|
98
|
+
return this.toResponse([ vector ], set.model, [ set.text ?? '' ]);
|
|
97
99
|
}
|
|
100
|
+
|
|
98
101
|
return vector;
|
|
99
102
|
}
|
|
100
103
|
|
|
@@ -120,6 +123,7 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
120
123
|
if (set.fullResponse) {
|
|
121
124
|
return this.toResponse(vectors, set.model, texts);
|
|
122
125
|
}
|
|
126
|
+
|
|
123
127
|
return vectors;
|
|
124
128
|
}
|
|
125
129
|
|
|
@@ -148,31 +152,38 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
148
152
|
protected deterministicVector(seed: string): number[] {
|
|
149
153
|
const hash: Buffer = crypto.createHash('sha256').update(seed).digest();
|
|
150
154
|
const vector: number[] = [];
|
|
155
|
+
|
|
151
156
|
for (let i = 0; i < this.dims; i++) {
|
|
152
157
|
const byte: number = hash[i % hash.length];
|
|
153
158
|
const mix: number = hash[(i * 7 + 3) % hash.length];
|
|
154
159
|
const combined: number = (byte ^ mix) & 0xff;
|
|
160
|
+
|
|
155
161
|
vector.push((combined / 255) * 2 - 1);
|
|
156
162
|
}
|
|
163
|
+
|
|
157
164
|
return this.l2Normalize(vector);
|
|
158
165
|
}
|
|
159
166
|
|
|
160
167
|
/** L2-normalizálás (egységhossz). Nulla-vektor → változatlan (elkerüli a 0-osztást). */
|
|
161
168
|
protected l2Normalize(vector: number[]): number[] {
|
|
162
169
|
let sumSq: number = 0;
|
|
170
|
+
|
|
163
171
|
for (const value of vector) {
|
|
164
172
|
sumSq += value * value;
|
|
165
173
|
}
|
|
166
174
|
const norm: number = Math.sqrt(sumSq);
|
|
175
|
+
|
|
167
176
|
if (!norm) {
|
|
168
177
|
return vector;
|
|
169
178
|
}
|
|
179
|
+
|
|
170
180
|
return vector.map((value) => value / norm);
|
|
171
181
|
}
|
|
172
182
|
|
|
173
183
|
/** OpenAI-alakú válasz-objektum a vektorokból (a `fullResponse=true` ághoz). */
|
|
174
184
|
protected toResponse(vectors: number[][], model: string, texts: string[]): CreateEmbeddingResponse {
|
|
175
185
|
const tokens: number = this.estimateTokens(texts);
|
|
186
|
+
|
|
176
187
|
return {
|
|
177
188
|
object: 'list',
|
|
178
189
|
model: model,
|
|
@@ -193,6 +204,7 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
193
204
|
issuer: string,
|
|
194
205
|
): void {
|
|
195
206
|
const tokens: number = this.estimateTokens(texts);
|
|
207
|
+
|
|
196
208
|
this.emitCostEvent({
|
|
197
209
|
callType: callType,
|
|
198
210
|
provider: this.costProvider,
|
|
@@ -211,9 +223,11 @@ export class DyNTS_AI_Embedding_MockService extends DyNTS_AI_Embedding_ServiceBa
|
|
|
211
223
|
/** Becsült (fake) token-szám: 4 char ≈ 1 token. */
|
|
212
224
|
protected estimateTokens(texts: string[]): number {
|
|
213
225
|
let chars: number = 0;
|
|
226
|
+
|
|
214
227
|
for (const text of texts) {
|
|
215
228
|
chars += (text ?? '').length;
|
|
216
229
|
}
|
|
230
|
+
|
|
217
231
|
return Math.max(1, Math.ceil(chars / 4));
|
|
218
232
|
}
|
|
219
233
|
}
|
|
@@ -60,12 +60,14 @@ export class DyNTS_AI_EmbeddingProvider_Registry {
|
|
|
60
60
|
});
|
|
61
61
|
case 'lmstudio': {
|
|
62
62
|
const baseUrl: string | undefined = options?.lmStudioSettings?.baseUrl ?? process.env.LMSTUDIO_BASE_URL;
|
|
63
|
+
|
|
63
64
|
if (!baseUrl) {
|
|
64
65
|
throw new Error(
|
|
65
66
|
'DyNTS_AI_EmbeddingProvider_Registry: the lmstudio provider was selected but no baseUrl '
|
|
66
67
|
+ 'was provided (set lmStudioSettings.baseUrl or the LMSTUDIO_BASE_URL env var).',
|
|
67
68
|
);
|
|
68
69
|
}
|
|
70
|
+
|
|
69
71
|
return new DyNTS_LMStudio_Embedding_ControlService({
|
|
70
72
|
baseUrl: baseUrl,
|
|
71
73
|
apiKey: options?.lmStudioSettings?.apiKey ?? process.env.LMSTUDIO_API_KEY,
|
|
@@ -86,6 +88,7 @@ export class DyNTS_AI_EmbeddingProvider_Registry {
|
|
|
86
88
|
*/
|
|
87
89
|
static normalizeKey(provider?: string): DyNTS_AI_EmbeddingProviderKey {
|
|
88
90
|
const raw: string = (provider ?? process.env.AI_EMBEDDING_PROVIDER ?? 'openai').trim().toLowerCase();
|
|
91
|
+
|
|
89
92
|
switch (raw) {
|
|
90
93
|
case 'openai':
|
|
91
94
|
case 'open-ai':
|
|
@@ -100,6 +103,7 @@ export class DyNTS_AI_EmbeddingProvider_Registry {
|
|
|
100
103
|
case 'fake':
|
|
101
104
|
case 'test':
|
|
102
105
|
return 'mock';
|
|
106
|
+
|
|
103
107
|
default:
|
|
104
108
|
throw new Error(
|
|
105
109
|
`DyNTS_AI_EmbeddingProvider_Registry: unknown embedding provider '${raw}'. `
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { CreateEmbeddingResponse } from 'openai/resources';
|
|
2
|
+
|
|
3
|
+
import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
|
|
2
4
|
import { DyFM_AI_Provider, DyFM_AI_ProviderCapabilities, DyFM_AI_Config } from '@futdevpro/fsm-dynamo/ai';
|
|
3
5
|
import { DyFM_DAI_EmbeddingInfo } from '@futdevpro/fsm-dynamo/ai/document-ai';
|
|
4
|
-
import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
|
|
5
6
|
|
|
6
|
-
import { DyNTS_AI_Embedding_ServiceBase } from './ai-embedding.service-base';
|
|
7
7
|
import { DyNTS_AI_CostEventCallback } from '../_models/interfaces/dynts-ai-cost-event-callback.interface';
|
|
8
8
|
import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
|
|
9
|
+
import { DyNTS_AI_Embedding_ServiceBase } from './ai-embedding.service-base';
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* Az LM Studio embedding control-service config-set-je. A `baseUrl` az OpenAI-kompatibilis
|
|
@@ -68,7 +69,8 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
68
69
|
*/
|
|
69
70
|
constructor(set: DyNTS_LMStudio_Embedding_Settings) {
|
|
70
71
|
super();
|
|
71
|
-
|
|
72
|
+
|
|
73
|
+
if (!set?.baseUrl?.trim().length) {
|
|
72
74
|
throw new DyFM_Error({
|
|
73
75
|
...this.getDefaultErrorSettings(
|
|
74
76
|
'constructor',
|
|
@@ -80,6 +82,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
80
82
|
}
|
|
81
83
|
this.baseUrl = this.trimTrailingSlashes(set.baseUrl.trim());
|
|
82
84
|
this.apiKey = set.apiKey;
|
|
85
|
+
|
|
83
86
|
if (set.onCostEvent) {
|
|
84
87
|
this.onCostEvent = set.onCostEvent;
|
|
85
88
|
}
|
|
@@ -93,6 +96,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
93
96
|
if (config?.baseURL) {
|
|
94
97
|
this.baseUrl = this.trimTrailingSlashes(config.baseURL);
|
|
95
98
|
}
|
|
99
|
+
|
|
96
100
|
if (config?.apiKey) {
|
|
97
101
|
this.apiKey = config.apiKey;
|
|
98
102
|
}
|
|
@@ -122,16 +126,18 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
122
126
|
const response: CreateEmbeddingResponse = await this.callEmbeddingsEndpoint(set.model, set.text, set.issuer);
|
|
123
127
|
const durationMs: number = Date.now() - start;
|
|
124
128
|
|
|
125
|
-
this.emitLmStudioCostEvent('embedding-single', set.model, [set.text], response, durationMs, set.issuer);
|
|
129
|
+
this.emitLmStudioCostEvent('embedding-single', set.model, [ set.text ], response, durationMs, set.issuer);
|
|
126
130
|
|
|
127
131
|
if (set.fullResponse) {
|
|
128
132
|
return response;
|
|
129
133
|
}
|
|
134
|
+
|
|
130
135
|
return response.data[0].embedding;
|
|
131
136
|
} catch (error) {
|
|
132
137
|
if (error instanceof DyFM_Error) {
|
|
133
138
|
throw error;
|
|
134
139
|
}
|
|
140
|
+
|
|
135
141
|
throw new DyFM_Error({
|
|
136
142
|
...this.getDefaultErrorSettings('createEmbedding', error, set.issuer),
|
|
137
143
|
errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LMS-ECS-CE0`,
|
|
@@ -168,11 +174,13 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
168
174
|
if (set.fullResponse) {
|
|
169
175
|
return response;
|
|
170
176
|
}
|
|
177
|
+
|
|
171
178
|
return response.data.map((item) => item.embedding);
|
|
172
179
|
} catch (error) {
|
|
173
180
|
if (error instanceof DyFM_Error) {
|
|
174
181
|
throw error;
|
|
175
182
|
}
|
|
183
|
+
|
|
176
184
|
throw new DyFM_Error({
|
|
177
185
|
...this.getDefaultErrorSettings('createEmbeddings', error, set.issuer),
|
|
178
186
|
errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LMS-ECS-CES0`,
|
|
@@ -198,17 +206,19 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
198
206
|
async testConnection(issuer: string): Promise<boolean> {
|
|
199
207
|
try {
|
|
200
208
|
const vectors: number[][] | CreateEmbeddingResponse = await this.createEmbeddings({
|
|
201
|
-
texts: ['ping'],
|
|
209
|
+
texts: [ 'ping' ],
|
|
202
210
|
model: this.defaultProbeModel(),
|
|
203
211
|
fullResponse: false,
|
|
204
212
|
issuer: issuer,
|
|
205
213
|
});
|
|
214
|
+
|
|
206
215
|
return Array.isArray(vectors) && vectors.length === 1 && vectors[0].length > 0;
|
|
207
216
|
} catch (error) {
|
|
208
217
|
DyFM_Log.error('DyNTS_LMStudio_Embedding_ControlService', 'testConnection', 'Connection test failed', {
|
|
209
218
|
error: error,
|
|
210
219
|
issuer: issuer,
|
|
211
220
|
});
|
|
221
|
+
|
|
212
222
|
return false;
|
|
213
223
|
}
|
|
214
224
|
}
|
|
@@ -229,6 +239,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
229
239
|
): Promise<CreateEmbeddingResponse> {
|
|
230
240
|
const url: string = `${this.baseUrl}/embeddings`;
|
|
231
241
|
const headers: { [key: string]: string } = { 'Content-Type': 'application/json' };
|
|
242
|
+
|
|
232
243
|
if (this.apiKey && this.apiKey.trim().length) {
|
|
233
244
|
headers.Authorization = `Bearer ${this.apiKey.trim()}`;
|
|
234
245
|
}
|
|
@@ -248,6 +259,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
248
259
|
}
|
|
249
260
|
|
|
250
261
|
let parsed: unknown;
|
|
262
|
+
|
|
251
263
|
try {
|
|
252
264
|
parsed = JSON.parse(rawText);
|
|
253
265
|
} catch {
|
|
@@ -257,6 +269,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
257
269
|
}
|
|
258
270
|
|
|
259
271
|
const expectedCount: number = Array.isArray(input) ? input.length : 1;
|
|
272
|
+
|
|
260
273
|
return this.normalizeResponse(parsed, expectedCount, url, model);
|
|
261
274
|
}
|
|
262
275
|
|
|
@@ -270,21 +283,25 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
270
283
|
throw new Error(`LM Studio embeddings: invalid response object | url=${url} | model=${modelId}`);
|
|
271
284
|
}
|
|
272
285
|
const dataRaw: unknown = Reflect.get(json, 'data');
|
|
286
|
+
|
|
273
287
|
if (!Array.isArray(dataRaw)) {
|
|
274
288
|
throw new Error(`LM Studio embeddings: missing data array | url=${url} | model=${modelId}`);
|
|
275
289
|
}
|
|
276
290
|
|
|
277
291
|
const data: CreateEmbeddingResponse['data'] = [];
|
|
278
292
|
let index: number = 0;
|
|
293
|
+
|
|
279
294
|
for (const item of dataRaw) {
|
|
280
295
|
if (item === null || typeof item !== 'object' || Array.isArray(item)) {
|
|
281
296
|
continue;
|
|
282
297
|
}
|
|
283
298
|
const embRaw: unknown = Reflect.get(item, 'embedding');
|
|
299
|
+
|
|
284
300
|
if (!Array.isArray(embRaw)) {
|
|
285
301
|
continue;
|
|
286
302
|
}
|
|
287
303
|
const vec: number[] = embRaw.filter((x: unknown): x is number => typeof x === 'number' && Number.isFinite(x));
|
|
304
|
+
|
|
288
305
|
if (vec.length) {
|
|
289
306
|
data.push({ object: 'embedding', embedding: vec, index: index });
|
|
290
307
|
index++;
|
|
@@ -345,9 +362,11 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
345
362
|
/** Becsült token-szám lokális endpoint-hoz (ha nincs `usage`): 4 char ≈ 1 token. */
|
|
346
363
|
protected estimateTokens(texts: string[]): number {
|
|
347
364
|
let chars: number = 0;
|
|
365
|
+
|
|
348
366
|
for (const text of texts) {
|
|
349
367
|
chars += (text ?? '').length;
|
|
350
368
|
}
|
|
369
|
+
|
|
351
370
|
return Math.max(1, Math.ceil(chars / 4));
|
|
352
371
|
}
|
|
353
372
|
|
|
@@ -364,6 +383,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
364
383
|
/** Rövid, biztonságos válasz-snapshot a hiba-üzenethez (max 300 char). */
|
|
365
384
|
protected snapshot(value: string): string {
|
|
366
385
|
const trimmed: string = (value ?? '').slice(0, 300);
|
|
386
|
+
|
|
367
387
|
return trimmed.length === 300 ? `${trimmed}…` : trimmed;
|
|
368
388
|
}
|
|
369
389
|
|
|
@@ -373,6 +393,7 @@ export class DyNTS_LMStudio_Embedding_ControlService extends DyNTS_AI_Embedding_
|
|
|
373
393
|
return 0;
|
|
374
394
|
}
|
|
375
395
|
const value: unknown = Reflect.get(obj, key);
|
|
396
|
+
|
|
376
397
|
return typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
|
377
398
|
}
|
|
378
399
|
}
|
package/src/_modules/ai/index.ts
CHANGED
|
@@ -5,6 +5,11 @@ export * from '@futdevpro/fsm-dynamo/ai';
|
|
|
5
5
|
export * from './_models/ai-input-interfaces';
|
|
6
6
|
export * from './_models/ai-test-generation-result.interface';
|
|
7
7
|
|
|
8
|
+
// Cost-event interfaces (BFR-AM-007) — a consumer (CCAP / FAM) ezekkel típusozza az `onCostEvent`
|
|
9
|
+
// callback-jét; ezért barrel-export kell (M3, FAM-REV bedrock-fix).
|
|
10
|
+
export * from './_models/interfaces/dynts-ai-cost-event.interface';
|
|
11
|
+
export * from './_models/interfaces/dynts-ai-cost-event-callback.interface';
|
|
12
|
+
|
|
8
13
|
// Abstract Services
|
|
9
14
|
export * from './_services/ai-provider.service-base';
|
|
10
15
|
export * from './_services/ai-llm.service-base';
|