@askdb/studio 0.2.0-beta.36 → 0.2.0-beta.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -4
- package/dist/bin.d.ts.map +1 -1
- package/dist/bin.js +16 -7
- package/dist/bin.js.map +1 -1
- package/dist/client/assets/index-Db2Xf11c.js +100 -0
- package/dist/client/index.html +1 -1
- package/dist/execute-registry.d.ts +4 -2
- package/dist/execute-registry.d.ts.map +1 -1
- package/dist/execute-registry.js +9 -3
- package/dist/execute-registry.js.map +1 -1
- package/dist/introspection.d.ts +6 -6
- package/dist/introspection.d.ts.map +1 -1
- package/dist/introspection.js +12 -69
- package/dist/introspection.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +213 -142
- package/dist/server.js.map +1 -1
- package/dist/setup.d.ts +3 -1
- package/dist/setup.d.ts.map +1 -1
- package/dist/setup.js +29 -33
- package/dist/setup.js.map +1 -1
- package/dist/shared/api.d.ts +11 -3
- package/dist/shared/api.d.ts.map +1 -1
- package/package.json +35 -36
- package/dist/client/assets/index-TAdxvl0x.js +0 -101
package/dist/server.js
CHANGED
|
@@ -5,21 +5,20 @@ import { extname, relative, join, resolve, dirname, sep } from "node:path";
|
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { generateText as defaultGenerateText } from "ai";
|
|
8
|
-
import { bootstrapAskDbEnv, getAskDbRuntimeConfig } from "@askdb/config";
|
|
8
|
+
import { ASKDB_AI_PROVIDERS, bootstrapAskDbEnv, getAskDbRuntimeConfig, } from "@askdb/config";
|
|
9
9
|
import { createAiRegistry, resolveReasoningEffort, } from "@askdb/ai";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import { openaiProvider } from "@askdb/ai-openai";
|
|
14
|
-
import { ask, formatSensitiveReference, isBuiltInDialectId, loadSchema, schemaHasSensitiveIdentifiers, SqlValidationError, validateSensitiveReferences, suggestEnrichment, tenantScopeSchema, normalizeTenantPolicy, parseTenantPolicyMarkdown, SchemaParseError, writeTenantPolicyMarkdown, tenantPolicyFrontmatterSchema, } from "@askdb/core";
|
|
15
|
-
import { buildSchemaIndex, chunkContentHash, chunkSchema, createAiSdkEmbedder, createFileStore, createMemoryStore, createPgvectorStore, createRetriever, loadChunkerSourcesFromDir, } from "@askdb/rag";
|
|
16
|
-
import { buildDefaultTableBody, buildFrontmatter, buildSuggestionContext, buildSuggestionTarget, buildTableDraft, loadWorkspace, replaceH2Section, replaceTableDescription, saveConcepts, saveTable, } from "@askdb/enrich";
|
|
10
|
+
import { ask, formatSchemaV2ForNlToSql, formatSensitiveReference, isBuiltInDialectId, loadSchema, schemaHasSensitiveIdentifiers, SensitiveEnrichmentTargetError, SqlValidationError, validateSensitiveReferences, suggestEnrichment, tenantScopeSchema, SchemaParseError, tenantPolicyFrontmatterSchema, } from "@askdb/core";
|
|
11
|
+
import { buildSchemaIndex, chunkContentHash, chunkSchema, aiSdkEmbedderId, createAiSdkEmbedder, createFileStore, createMemoryStore, createPgvectorStore, createRetriever, detectEmbeddingDimensions, loadChunkerSourcesFromDir, PgvectorDimensionMismatchError, } from "@askdb/rag";
|
|
12
|
+
import { buildDefaultTableBody, buildFrontmatter, buildSuggestionContext, buildSuggestionTarget, buildTableDraft, loadWorkspace, replaceH2Section, replaceTableDescription, saveConcepts, saveTable, saveTenantPolicy, } from "@askdb/enrich";
|
|
17
13
|
import { EXECUTE_DRIVER_REGISTRY, executeDialectFor, isDriverInstalled, isStudioExecuteProvider, validateExecuteSql, } from "./execute-registry.js";
|
|
18
14
|
import { lockfilePackageManager, packageManagerAddArgs, packageManagerSpawnSpec, } from "./package-manager.js";
|
|
19
15
|
import { resolveStudioIntrospectionPlan, runStudioIntrospection } from "./introspection.js";
|
|
20
16
|
import { probeSetupState, writeSetupConfig, SetupError } from "./setup.js";
|
|
21
17
|
import { checkApiRequest, checkHost, createStudioSessionToken, injectSessionToken, } from "./request-guard.js";
|
|
22
|
-
|
|
18
|
+
// Batteries-included surface: every built-in provider is registered, and each
|
|
19
|
+
// loads its @ai-sdk/* package only when first used, so env config alone
|
|
20
|
+
// selects the provider.
|
|
21
|
+
const ai = createAiRegistry();
|
|
23
22
|
const DEFAULT_CLIENT_DIR = fileURLToPath(new URL("./client/", import.meta.url));
|
|
24
23
|
let clientDirForTests;
|
|
25
24
|
/** @internal Tests only — serve client assets from another directory. */
|
|
@@ -31,7 +30,6 @@ function clientDir() {
|
|
|
31
30
|
}
|
|
32
31
|
const STUDIO_RAG_MOCK_DIMENSIONS = 64;
|
|
33
32
|
const STUDIO_RAG_MOCK_EMBEDDER_ID = `studio:mock-lexical-${STUDIO_RAG_MOCK_DIMENSIONS}`;
|
|
34
|
-
const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
35
33
|
let studioPgvectorStoreFactoryForTests;
|
|
36
34
|
export function setStudioPgvectorStoreFactoryForTests(factory) {
|
|
37
35
|
studioPgvectorStoreFactoryForTests = factory;
|
|
@@ -124,7 +122,7 @@ export function createStudioServer(options) {
|
|
|
124
122
|
if (req.method === "POST" && url.pathname === "/api/tenant-policy") {
|
|
125
123
|
const body = await readJson(req);
|
|
126
124
|
const { frontmatter, bodyText } = parseTenantPolicyBody(body);
|
|
127
|
-
|
|
125
|
+
saveTenantPolicyDraft(state, frontmatter, bodyText);
|
|
128
126
|
return writeJson(res, 200, serializeWorkspace(requireWorkspace(state)));
|
|
129
127
|
}
|
|
130
128
|
if (req.method === "POST" && url.pathname === "/api/suggest-tenant-policy") {
|
|
@@ -134,7 +132,7 @@ export function createStudioServer(options) {
|
|
|
134
132
|
if (req.method === "POST" && url.pathname === "/api/suggest") {
|
|
135
133
|
const body = await readJson(req);
|
|
136
134
|
const source = parseSuggestSource(body);
|
|
137
|
-
const candidates = await suggestForSource(
|
|
135
|
+
const candidates = await suggestForSource(state, source);
|
|
138
136
|
return writeJson(res, 200, { candidates });
|
|
139
137
|
}
|
|
140
138
|
if (req.method === "GET" && url.pathname === "/api/rag/status") {
|
|
@@ -224,8 +222,8 @@ export function serializeWorkspace(workspace) {
|
|
|
224
222
|
dialect,
|
|
225
223
|
warnings: workspace.warnings,
|
|
226
224
|
aiConfigured: Boolean(aiConfig),
|
|
227
|
-
model: aiConfig?.model ?? "
|
|
228
|
-
aiProvider: aiConfig?.provider ??
|
|
225
|
+
model: aiConfig?.model ?? rt.ai.language.model ?? "provider default",
|
|
226
|
+
aiProvider: aiConfig?.provider ?? rt.ai.language.provider,
|
|
229
227
|
tables: workspace.tables.map((table) => {
|
|
230
228
|
const draft = buildTableDraft(table.physical, table.parsed);
|
|
231
229
|
return {
|
|
@@ -373,7 +371,8 @@ function parseSetupConfigBody(body) {
|
|
|
373
371
|
throw new StudioHttpError(400, "Request body must be a JSON object.");
|
|
374
372
|
}
|
|
375
373
|
const databases = ["postgres", "mysql", "sqlite", "sqlserver", "prisma"];
|
|
376
|
-
|
|
374
|
+
// Every provider with an askdb.config.* branch; @askdb/client's provider-config-drift test asserts this matches @askdb/ai's built-in table.
|
|
375
|
+
const aiProviders = ASKDB_AI_PROVIDERS;
|
|
377
376
|
const ragStores = ["file", "memory", "pgvector"];
|
|
378
377
|
const executeProviders = ["postgres", "mysql", "sqlite", "sqlserver"];
|
|
379
378
|
if (typeof body.database !== "string" || !databases.includes(body.database)) {
|
|
@@ -422,23 +421,17 @@ function saveConceptsDraft(state, concepts) {
|
|
|
422
421
|
saveConcepts(requireWorkspace(state), { concepts });
|
|
423
422
|
state.workspace = loadWorkspace(state.schemaDir);
|
|
424
423
|
}
|
|
425
|
-
function
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
const md = writeTenantPolicyMarkdown(frontmatter, body);
|
|
429
|
-
// Run core's own load-time validation before writing: a policy that `loadSchema` would
|
|
430
|
-
// reject (e.g. two root labels deriving one placeholder) must not reach disk, or every
|
|
431
|
-
// later request fails until the file is fixed by hand.
|
|
424
|
+
function saveTenantPolicyDraft(state, frontmatter, body) {
|
|
425
|
+
// `saveTenantPolicy` refuses a policy `loadSchema` would reject (e.g. two root labels
|
|
426
|
+
// deriving one placeholder) before writing; that's the caller's mistake, so 400.
|
|
432
427
|
try {
|
|
433
|
-
|
|
434
|
-
normalizeTenantPolicy(parseTenantPolicyMarkdown(md, filePath), new Set(physical.tables.map((table) => table.id)), new Set(physical.tables.flatMap((table) => table.columns.map((column) => column.id))));
|
|
428
|
+
saveTenantPolicy(requireWorkspace(state), frontmatter, body);
|
|
435
429
|
}
|
|
436
430
|
catch (error) {
|
|
437
431
|
if (error instanceof SchemaParseError)
|
|
438
432
|
throw new StudioHttpError(400, error.message);
|
|
439
433
|
throw error;
|
|
440
434
|
}
|
|
441
|
-
writeFileSync(filePath, md, "utf8");
|
|
442
435
|
state.workspace = loadWorkspace(state.schemaDir);
|
|
443
436
|
}
|
|
444
437
|
function parseTenantPolicyBody(body) {
|
|
@@ -472,7 +465,7 @@ function saveDraft(state, tableId, draft) {
|
|
|
472
465
|
saveTable(workspace, tableId, frontmatter, body);
|
|
473
466
|
state.workspace = loadWorkspace(state.schemaDir);
|
|
474
467
|
}
|
|
475
|
-
async function suggestForSource(
|
|
468
|
+
async function suggestForSource(state, source) {
|
|
476
469
|
const rt = getAskDbRuntimeConfig();
|
|
477
470
|
const aiConfig = ai.resolveAiConfig(rt.ai.aiEnv);
|
|
478
471
|
if (!aiConfig) {
|
|
@@ -483,37 +476,108 @@ async function suggestForSource(workspace, source) {
|
|
|
483
476
|
const providerOptions = reasoningEffort
|
|
484
477
|
? ai.resolveProviderOptions(aiConfig, { reasoningEffort })
|
|
485
478
|
: undefined;
|
|
486
|
-
|
|
479
|
+
// `loadSchema()` carries effective sensitivity (front-matter escalations included), and the
|
|
480
|
+
// prompt follows `modes.omitSensitiveFromPrompt` as `ask()` does (#625, ADR 0026).
|
|
481
|
+
requireWorkspace(state);
|
|
482
|
+
const schema = loadSchema(state.schemaDir);
|
|
483
|
+
let candidates;
|
|
484
|
+
try {
|
|
485
|
+
candidates = await suggestEnrichment(buildSuggestionTarget(schema, source), buildSuggestionContext(schema, source.tableId), model, {
|
|
486
|
+
omitSensitiveIdentifiersFromPrompt: rt.modes.omitSensitiveFromPrompt,
|
|
487
|
+
...(providerOptions ? { providerOptions } : {}),
|
|
488
|
+
});
|
|
489
|
+
}
|
|
490
|
+
catch (error) {
|
|
491
|
+
if (error instanceof SensitiveEnrichmentTargetError) {
|
|
492
|
+
throw new StudioHttpError(400, "This column is sensitive, and `modes.omitSensitiveFromPrompt` keeps sensitive columns out of the model's prompt, so Studio can't suggest for it.");
|
|
493
|
+
}
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
487
496
|
return candidates.map((candidate) => ({ text: candidate.text }));
|
|
488
497
|
}
|
|
498
|
+
/**
|
|
499
|
+
* The schema as the tenant-policy draft sends it to the model, with the sensitivity rules
|
|
500
|
+
* `ask()` applies (#324): `formatSchemaV2ForNlToSql` tags sensitive columns, or withholds
|
|
501
|
+
* them when `omitSensitive` is on. Only the physical layer goes in (names, types, keys and
|
|
502
|
+
* foreign keys), never descriptions, aliases or common query language, and untracked tables
|
|
503
|
+
* stay out as they do for `ask()`. A foreign key is listed only when both of its columns are.
|
|
504
|
+
* `warnings` names the primary and foreign keys withheld as sensitive, which the draft can't
|
|
505
|
+
* scope through.
|
|
506
|
+
*/
|
|
507
|
+
function tenantPolicySchemaForPrompt(schema, omitSensitive) {
|
|
508
|
+
const tracked = schema.tables.filter((table) => table.tracked !== false);
|
|
509
|
+
const physical = {
|
|
510
|
+
schemaId: schema.schemaId,
|
|
511
|
+
warnings: [],
|
|
512
|
+
tables: tracked.map((table) => ({
|
|
513
|
+
id: table.id,
|
|
514
|
+
name: table.name,
|
|
515
|
+
schema: table.schema,
|
|
516
|
+
sensitive: table.sensitive,
|
|
517
|
+
columns: table.columns.map((column) => ({
|
|
518
|
+
id: column.id,
|
|
519
|
+
name: column.name,
|
|
520
|
+
type: column.type,
|
|
521
|
+
nullable: column.nullable,
|
|
522
|
+
primaryKey: column.primaryKey,
|
|
523
|
+
sensitive: column.sensitive,
|
|
524
|
+
})),
|
|
525
|
+
})),
|
|
526
|
+
};
|
|
527
|
+
const { ddl } = formatSchemaV2ForNlToSql(physical, { omitSensitiveIdentifiersFromPrompt: omitSensitive });
|
|
528
|
+
// The formatter prints no foreign keys yet, so Studio lists them, mirroring the formatter's
|
|
529
|
+
// omission rule (packages/core/src/schema/v2/format.ts): a column is withheld when omission
|
|
530
|
+
// is on and it is sensitive, which `loadSchema()` sets on every column of a sensitive table.
|
|
531
|
+
// When core prints foreign keys (#349, step 5), use its lines and delete this block (ADR 0021).
|
|
532
|
+
const names = new Map();
|
|
533
|
+
const withheld = new Set();
|
|
534
|
+
for (const table of schema.tables) {
|
|
535
|
+
for (const column of table.columns) {
|
|
536
|
+
names.set(column.id, `${table.schema}.${table.name}.${column.name}`);
|
|
537
|
+
if (omitSensitive && column.sensitive)
|
|
538
|
+
withheld.add(column.id);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
const nameOf = (id) => names.get(id) ?? id;
|
|
542
|
+
const listed = new Set();
|
|
543
|
+
const withheldKeys = [];
|
|
544
|
+
for (const table of tracked) {
|
|
545
|
+
for (const column of table.columns) {
|
|
546
|
+
if (!withheld.has(column.id))
|
|
547
|
+
listed.add(column.id);
|
|
548
|
+
else if (column.primaryKey)
|
|
549
|
+
withheldKeys.push(`${nameOf(column.id)} (primary key)`);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
const foreignKeys = [];
|
|
553
|
+
for (const fk of tracked.flatMap((table) => table.relationships ?? [])) {
|
|
554
|
+
if (listed.has(fk.from) && listed.has(fk.to))
|
|
555
|
+
foreignKeys.push(`- ${fk.from} -> ${fk.to}`);
|
|
556
|
+
else if (withheld.has(fk.from) || withheld.has(fk.to)) {
|
|
557
|
+
withheldKeys.push(`${nameOf(fk.from)} → ${nameOf(fk.to)} (foreign key)`);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
const warnings = withheldKeys.length > 0
|
|
561
|
+
? [
|
|
562
|
+
`Withheld from the model because they're marked sensitive: ${withheldKeys.join(", ")}. ` +
|
|
563
|
+
"If any of them carries tenant scope, add it to the policy by hand.",
|
|
564
|
+
]
|
|
565
|
+
: [];
|
|
566
|
+
return {
|
|
567
|
+
schemaId: schema.schemaId,
|
|
568
|
+
schemaDdl: foreignKeys.length > 0 ? `${ddl}\n\nForeign keys:\n${foreignKeys.join("\n")}` : ddl,
|
|
569
|
+
tableIds: tracked.map((table) => table.id),
|
|
570
|
+
warnings,
|
|
571
|
+
};
|
|
572
|
+
}
|
|
489
573
|
async function suggestTenantPolicyDraft(state) {
|
|
490
574
|
const rt = getAskDbRuntimeConfig();
|
|
491
575
|
const model = await ai.createLanguageModelFromEnv(rt.ai.aiEnv);
|
|
492
576
|
if (!model) {
|
|
493
577
|
throw new StudioHttpError(400, ai.keyMissingMessage("AI tenant policy suggestion"));
|
|
494
578
|
}
|
|
495
|
-
|
|
496
|
-
const
|
|
497
|
-
const schemaId = workspace.physical.schemaId;
|
|
498
|
-
const schemaDdl = workspace.tables
|
|
499
|
-
.map((t) => {
|
|
500
|
-
const cols = t.physical.columns
|
|
501
|
-
.map((c) => {
|
|
502
|
-
const parts = [c.name, c.type];
|
|
503
|
-
if (c.primaryKey)
|
|
504
|
-
parts.push("PRIMARY KEY");
|
|
505
|
-
if (!c.nullable)
|
|
506
|
-
parts.push("NOT NULL");
|
|
507
|
-
return ` ${parts.join(" ")}`;
|
|
508
|
-
})
|
|
509
|
-
.join(",\n");
|
|
510
|
-
const rels = (t.physical.relationships ?? [])
|
|
511
|
-
.map((r) => ` -- FK: ${r.from} -> ${r.to}`)
|
|
512
|
-
.join("\n");
|
|
513
|
-
return `CREATE TABLE ${t.physical.schema}.${t.physical.name} (\n${cols}\n);\n${rels}`.trim();
|
|
514
|
-
})
|
|
515
|
-
.join("\n\n");
|
|
516
|
-
const tableIds = workspace.tables.map((t) => t.physical.id);
|
|
579
|
+
requireWorkspace(state);
|
|
580
|
+
const { schemaId, schemaDdl, tableIds, warnings } = tenantPolicySchemaForPrompt(loadSchema(state.schemaDir), rt.modes.omitSensitiveFromPrompt);
|
|
517
581
|
const systemPrompt = `You are an expert database architect specializing in multi-tenant SaaS systems.
|
|
518
582
|
You analyze database schemas and produce a tenant-policy configuration in a specific JSON format.
|
|
519
583
|
|
|
@@ -572,8 +636,7 @@ If the schema has no clear multi-tenant pattern, still make your best guess at w
|
|
|
572
636
|
if (!validation.success) {
|
|
573
637
|
throw new StudioHttpError(500, `AI-generated tenant policy frontmatter is invalid: ${validation.error.issues.map((i) => i.message).join("; ")}`);
|
|
574
638
|
}
|
|
575
|
-
|
|
576
|
-
return parsed;
|
|
639
|
+
return { frontmatter: validation.data, body: parsed.body, warnings };
|
|
577
640
|
}
|
|
578
641
|
async function askSampleQuestion(state, options) {
|
|
579
642
|
const rt = getAskDbRuntimeConfig();
|
|
@@ -604,6 +667,9 @@ async function askSampleQuestion(state, options) {
|
|
|
604
667
|
model,
|
|
605
668
|
dialect,
|
|
606
669
|
explain: true,
|
|
670
|
+
// `ask()` doesn't read config, so each host applies `modes.omitSensitiveFromPrompt`
|
|
671
|
+
// itself, as `@askdb/client` does (#601).
|
|
672
|
+
...(rt.modes.omitSensitiveFromPrompt ? { omitSensitiveIdentifiersFromNlToSqlPrompt: true } : {}),
|
|
607
673
|
...(options.tenantScope ? { tenantScope: options.tenantScope } : {}),
|
|
608
674
|
...(options.tenantSqlMode ? { tenantSqlMode: options.tenantSqlMode } : {}),
|
|
609
675
|
...(ragIndex
|
|
@@ -659,14 +725,17 @@ async function getRagStatus(state) {
|
|
|
659
725
|
const sources = loadChunkerSourcesFromDir(state.schemaDir);
|
|
660
726
|
const chunkResult = chunkSchema(sources);
|
|
661
727
|
const lockPath = join(state.schemaDir, "schema.lock.json");
|
|
662
|
-
const lock =
|
|
728
|
+
const lock = resolveStudioRagStoreConfig(state).kind === "memory"
|
|
729
|
+
? state.ragMemoryIndex
|
|
730
|
+
: readOptionalJson(lockPath);
|
|
663
731
|
const currentHashes = Object.fromEntries(chunkResult.chunks.map((chunk) => [chunk.id, chunkContentHash(chunk.text)]));
|
|
664
732
|
const lockHashes = lock?.hashes ?? {};
|
|
665
733
|
const hashIds = Object.keys(currentHashes);
|
|
666
|
-
const store = await openStudioRagStore(state, config.dimensions);
|
|
734
|
+
const store = await openStudioRagStore(state, { dimensions: config.dimensions });
|
|
667
735
|
try {
|
|
668
736
|
const chunksIndexed = await countStudioRagStoreChunks(store, sources.schema.schemaId);
|
|
669
737
|
const stale = !lock ||
|
|
738
|
+
lock.incomplete === true ||
|
|
670
739
|
lock.embedderId !== config.embedderId ||
|
|
671
740
|
Object.keys(lockHashes).length !== hashIds.length ||
|
|
672
741
|
hashIds.some((id) => lockHashes[id] !== currentHashes[id]) ||
|
|
@@ -707,8 +776,9 @@ async function getRagStatus(state) {
|
|
|
707
776
|
updatedAt: lock?.updatedAt ?? null,
|
|
708
777
|
chunksTotal: chunkResult.chunks.length,
|
|
709
778
|
chunksIndexed,
|
|
710
|
-
|
|
711
|
-
|
|
779
|
+
// The width the last index build learned from the model, else the one configured.
|
|
780
|
+
dimensions: lock?.dimensions ?? config.dimensions ?? null,
|
|
781
|
+
expectedDimensions: config.dimensions ?? null,
|
|
712
782
|
sensitiveExcluded: chunkResult.stats.sensitiveExcluded,
|
|
713
783
|
sensitiveIncluded: chunkResult.stats.sensitiveIncluded,
|
|
714
784
|
files: fileArtifacts,
|
|
@@ -721,27 +791,40 @@ async function getRagStatus(state) {
|
|
|
721
791
|
async function indexRag(state) {
|
|
722
792
|
const config = resolveStudioRagEmbedderConfig();
|
|
723
793
|
if (!config.configured) {
|
|
724
|
-
throw new StudioHttpError(400, studioRagAiSdkKeyMissingMessage());
|
|
794
|
+
throw new StudioHttpError(400, studioRagAiSdkKeyMissingMessage(config));
|
|
725
795
|
}
|
|
726
796
|
const usage = createRequestUsageCollector();
|
|
727
797
|
clearIncompatibleRagStore(state, config);
|
|
728
798
|
const sources = loadChunkerSourcesFromDir(state.schemaDir);
|
|
729
|
-
const
|
|
799
|
+
const embedder = await createStudioRagEmbedder(config, usage);
|
|
730
800
|
let result;
|
|
801
|
+
let store;
|
|
731
802
|
try {
|
|
803
|
+
// A new pgvector table needs its width up front: the configured one, else the model's own.
|
|
804
|
+
const dimensions = config.dimensions ??
|
|
805
|
+
(resolveStudioRagStoreConfig(state).kind === "pgvector" ? await detectEmbeddingDimensions(embedder) : undefined);
|
|
806
|
+
store = await openStudioRagStore(state, { dimensions, provision: true });
|
|
732
807
|
result = await buildSchemaIndex({
|
|
733
808
|
schema: sources,
|
|
734
|
-
embedder
|
|
809
|
+
embedder,
|
|
735
810
|
store: store.store,
|
|
736
811
|
embedderId: config.embedderId,
|
|
737
812
|
lockFilePath: join(state.schemaDir, "schema.lock.json"),
|
|
738
813
|
});
|
|
814
|
+
if (store.kind === "memory") {
|
|
815
|
+
state.ragMemoryIndex = {
|
|
816
|
+
embedderId: config.embedderId,
|
|
817
|
+
updatedAt: new Date().toISOString(),
|
|
818
|
+
dimensions: store.store.describe?.().dimensions,
|
|
819
|
+
hashes: Object.fromEntries(result.chunks.map((chunk) => [chunk.id, chunkContentHash(chunk.text)])),
|
|
820
|
+
};
|
|
821
|
+
}
|
|
739
822
|
}
|
|
740
823
|
catch (error) {
|
|
741
824
|
throw formatStudioRagOperationError(error, config);
|
|
742
825
|
}
|
|
743
826
|
finally {
|
|
744
|
-
await store
|
|
827
|
+
await store?.dispose();
|
|
745
828
|
}
|
|
746
829
|
return {
|
|
747
830
|
status: await getRagStatus(state),
|
|
@@ -779,7 +862,7 @@ async function queryRag(state, query) {
|
|
|
779
862
|
async function createCurrentStudioRagIndex(state, action, usage) {
|
|
780
863
|
const config = resolveStudioRagEmbedderConfig();
|
|
781
864
|
if (!config.configured) {
|
|
782
|
-
throw new StudioHttpError(400, studioRagAiSdkKeyMissingMessage());
|
|
865
|
+
throw new StudioHttpError(400, studioRagAiSdkKeyMissingMessage(config));
|
|
783
866
|
}
|
|
784
867
|
const status = (await getRagStatus(state));
|
|
785
868
|
if (!status.hasIndex) {
|
|
@@ -791,7 +874,7 @@ async function createCurrentStudioRagIndex(state, action, usage) {
|
|
|
791
874
|
if (!status.schemaId || typeof status.chunksTotal !== "number") {
|
|
792
875
|
throw new StudioHttpError(500, "Studio RAG status is missing schema metadata.");
|
|
793
876
|
}
|
|
794
|
-
const store = await openStudioRagStore(state, config.dimensions);
|
|
877
|
+
const store = await openStudioRagStore(state, { dimensions: config.dimensions });
|
|
795
878
|
return {
|
|
796
879
|
config,
|
|
797
880
|
status: {
|
|
@@ -809,22 +892,26 @@ async function createCurrentStudioRagIndex(state, action, usage) {
|
|
|
809
892
|
}
|
|
810
893
|
function resolveStudioRagStoreConfig(state) {
|
|
811
894
|
const rt = getAskDbRuntimeConfig();
|
|
812
|
-
const kind = rt.
|
|
895
|
+
const kind = rt.rag.store;
|
|
813
896
|
if (kind === "memory")
|
|
814
897
|
return { kind };
|
|
815
898
|
if (kind === "file") {
|
|
816
|
-
const basePath = rt.
|
|
899
|
+
const basePath = rt.rag.storeConfig.file?.basePath?.trim();
|
|
817
900
|
return { kind, basePath: basePath ? resolve(basePath) : join(state.schemaDir, "schema") };
|
|
818
901
|
}
|
|
819
902
|
const connectionString = pickFlat(rt.flat, "ASKDB_PGVECTOR_URL");
|
|
820
903
|
return {
|
|
821
904
|
kind,
|
|
822
905
|
connectionString,
|
|
823
|
-
table: rt.
|
|
906
|
+
table: rt.rag.storeConfig.pgvector?.table?.trim() || undefined,
|
|
824
907
|
indexStrategy: pickFlat(rt.flat, "ASKDB_PGVECTOR_INDEX_STRATEGY"),
|
|
825
908
|
};
|
|
826
909
|
}
|
|
827
|
-
|
|
910
|
+
/**
|
|
911
|
+
* Opens the configured store. `provision` creates the pgvector table when it's missing, at
|
|
912
|
+
* `dimensions`; without it, a pgvector store only reads and writes an existing table.
|
|
913
|
+
*/
|
|
914
|
+
async function openStudioRagStore(state, { dimensions, provision = false }) {
|
|
828
915
|
const config = resolveStudioRagStoreConfig(state);
|
|
829
916
|
if (config.kind === "memory") {
|
|
830
917
|
state.ragMemoryStore ??= createMemoryStore();
|
|
@@ -852,10 +939,19 @@ async function openStudioRagStore(state, dimensions) {
|
|
|
852
939
|
const store = pgvectorFactory({
|
|
853
940
|
connectionString: config.connectionString,
|
|
854
941
|
table: config.table,
|
|
855
|
-
dimensions,
|
|
942
|
+
...(dimensions !== undefined ? { dimensions } : {}),
|
|
856
943
|
...(config.indexStrategy ? { indexStrategy: config.indexStrategy } : {}),
|
|
857
944
|
});
|
|
858
|
-
|
|
945
|
+
if (provision) {
|
|
946
|
+
try {
|
|
947
|
+
await store.ensureSchema();
|
|
948
|
+
}
|
|
949
|
+
catch (error) {
|
|
950
|
+
// The caller never receives this store, so close its pool here (a width refusal has opened it).
|
|
951
|
+
await store.close().catch(() => { });
|
|
952
|
+
throw error;
|
|
953
|
+
}
|
|
954
|
+
}
|
|
859
955
|
return {
|
|
860
956
|
kind: "pgvector",
|
|
861
957
|
store,
|
|
@@ -867,6 +963,10 @@ async function openStudioRagStore(state, dimensions) {
|
|
|
867
963
|
};
|
|
868
964
|
}
|
|
869
965
|
async function countStudioRagStoreChunks(store, schemaId) {
|
|
966
|
+
// A pgvector table that doesn't exist yet holds nothing; the first index build creates it.
|
|
967
|
+
if (typeof store.store.tableDimensions === "function" && (await store.store.tableDimensions()) === undefined) {
|
|
968
|
+
return 0;
|
|
969
|
+
}
|
|
870
970
|
if (typeof store.store.count === "function") {
|
|
871
971
|
return store.store.count({ schemaId });
|
|
872
972
|
}
|
|
@@ -879,16 +979,9 @@ function pickFlat(flat, key) {
|
|
|
879
979
|
const value = flat[key];
|
|
880
980
|
return typeof value === "string" && value.trim() !== "" ? value.trim() : undefined;
|
|
881
981
|
}
|
|
882
|
-
function pickEnv(env, key) {
|
|
883
|
-
const v = env[key];
|
|
884
|
-
return typeof v === "string" && v.trim() !== "" ? v.trim() : undefined;
|
|
885
|
-
}
|
|
886
982
|
function resolveStudioRagEmbedderConfig() {
|
|
887
|
-
const
|
|
888
|
-
|
|
889
|
-
const explicitKind = pickEnv(base, "ASKDB_RAG_EMBEDDER");
|
|
890
|
-
const kind = explicitKind?.toLowerCase();
|
|
891
|
-
if (kind === "mock") {
|
|
983
|
+
const embedding = getAskDbRuntimeConfig().ai.embedding;
|
|
984
|
+
if (!embedding) {
|
|
892
985
|
return {
|
|
893
986
|
kind: "mock",
|
|
894
987
|
embedderId: STUDIO_RAG_MOCK_EMBEDDER_ID,
|
|
@@ -897,67 +990,38 @@ function resolveStudioRagEmbedderConfig() {
|
|
|
897
990
|
label: "Mock lexical",
|
|
898
991
|
};
|
|
899
992
|
}
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
const
|
|
904
|
-
const
|
|
905
|
-
|
|
906
|
-
modelDefault: DEFAULT_EMBEDDING_MODEL,
|
|
907
|
-
});
|
|
908
|
-
if (!aiConfig && kind === undefined) {
|
|
909
|
-
return {
|
|
910
|
-
kind: "mock",
|
|
911
|
-
embedderId: STUDIO_RAG_MOCK_EMBEDDER_ID,
|
|
912
|
-
dimensions: STUDIO_RAG_MOCK_DIMENSIONS,
|
|
913
|
-
configured: true,
|
|
914
|
-
label: "Mock lexical",
|
|
915
|
-
};
|
|
916
|
-
}
|
|
917
|
-
const provider = aiConfig?.provider ?? fallbackStudioRagProvider(kind, base);
|
|
918
|
-
const model = aiConfig?.model ?? DEFAULT_EMBEDDING_MODEL;
|
|
919
|
-
const dimensionOverride = readPositiveIntegerEnv(pickEnv(base, "ASKDB_RAG_EMBEDDER_DIMENSIONS"));
|
|
920
|
-
const dimensions = dimensionOverride ?? defaultEmbeddingDimensions(model);
|
|
993
|
+
// The env holds the embedding connection only, so no other provider's key can reach it (#345).
|
|
994
|
+
const aiConfig = ai.resolveEmbeddingConfig(embedding.env);
|
|
995
|
+
// The adapter's canonical name (`foundry` resolves to `azure`) keeps existing index ids stable.
|
|
996
|
+
const provider = aiConfig?.provider ?? embedding.provider;
|
|
997
|
+
const model = embedding.model;
|
|
998
|
+
const dimensions = embedding.dimensions;
|
|
921
999
|
return {
|
|
922
1000
|
kind: "ai-sdk",
|
|
923
1001
|
provider,
|
|
924
|
-
|
|
1002
|
+
configuredProvider: embedding.provider,
|
|
1003
|
+
connection: embedding.connection,
|
|
1004
|
+
embedderId: aiSdkEmbedderId({ provider, model, dimensions }),
|
|
925
1005
|
dimensions,
|
|
926
1006
|
configured: Boolean(aiConfig),
|
|
927
1007
|
label: `AI SDK (${provider})`,
|
|
928
1008
|
model,
|
|
929
1009
|
baseUrl: aiConfig?.baseURL,
|
|
930
1010
|
aiConfig,
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
}
|
|
934
|
-
function buildStudioRagEmbeddingEnv(kind, base) {
|
|
935
|
-
const apiKeyOverride = pickEnv(base, "ASKDB_RAG_EMBEDDER_API_KEY");
|
|
936
|
-
const baseUrlOverride = pickEnv(base, "ASKDB_RAG_EMBEDDER_BASE_URL");
|
|
937
|
-
return {
|
|
938
|
-
...base,
|
|
939
|
-
...(kind === "openai" ? { ASKDB_AI_PROVIDER: "openai" } : {}),
|
|
940
|
-
...(apiKeyOverride ? { ASKDB_AI_API_KEY: apiKeyOverride } : {}),
|
|
941
|
-
...(baseUrlOverride ? { ASKDB_AI_BASE_URL: baseUrlOverride } : {}),
|
|
1011
|
+
// Only an explicit ai.embedding.dimensions is sent; otherwise the model's own width applies.
|
|
1012
|
+
requestDimensions: embedding.dimensions,
|
|
942
1013
|
};
|
|
943
1014
|
}
|
|
944
|
-
function
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
return raw === "azure" || raw === "azure-openai" || raw === "foundry"
|
|
949
|
-
? "azure"
|
|
950
|
-
: "openai";
|
|
951
|
-
}
|
|
952
|
-
function studioRagAiSdkKeyMissingMessage() {
|
|
953
|
-
return ("Studio RAG AI SDK embeddings require a configured AI provider key. " +
|
|
954
|
-
"Set ASKDB_AI_API_KEY or the provider-native key, or set ASKDB_RAG_EMBEDDER=mock for the local lexical embedder.");
|
|
1015
|
+
function studioRagAiSdkKeyMissingMessage(config) {
|
|
1016
|
+
return (`Studio RAG embeddings need an API key on the ai.embedding connection ` +
|
|
1017
|
+
`("${config.connection}" in ai.providerConfig.${config.configuredProvider}). ` +
|
|
1018
|
+
'Set it in askdb.config.*, or set rag.embedder: "mock" for the local lexical embedder.');
|
|
955
1019
|
}
|
|
956
1020
|
async function createStudioRagEmbedder(config, usage) {
|
|
957
1021
|
if (config.kind === "mock")
|
|
958
1022
|
return createStudioMockEmbedder(config.dimensions);
|
|
959
1023
|
if (!config.aiConfig) {
|
|
960
|
-
throw new StudioHttpError(400, studioRagAiSdkKeyMissingMessage());
|
|
1024
|
+
throw new StudioHttpError(400, studioRagAiSdkKeyMissingMessage(config));
|
|
961
1025
|
}
|
|
962
1026
|
const model = await ai.createEmbeddingModel(config.aiConfig, {
|
|
963
1027
|
dimensions: config.requestDimensions,
|
|
@@ -977,6 +1041,9 @@ async function createStudioRagEmbedder(config, usage) {
|
|
|
977
1041
|
function formatStudioRagOperationError(error, config) {
|
|
978
1042
|
if (error instanceof StudioHttpError)
|
|
979
1043
|
return error;
|
|
1044
|
+
if (error instanceof PgvectorDimensionMismatchError) {
|
|
1045
|
+
return new StudioHttpError(409, studioRagWidthMismatchMessage(error, config));
|
|
1046
|
+
}
|
|
980
1047
|
if (config.kind === "mock") {
|
|
981
1048
|
return new StudioHttpError(500, error instanceof Error ? error.message : String(error));
|
|
982
1049
|
}
|
|
@@ -985,7 +1052,7 @@ function formatStudioRagOperationError(error, config) {
|
|
|
985
1052
|
return new StudioHttpError(500, error instanceof Error ? error.message : String(error));
|
|
986
1053
|
}
|
|
987
1054
|
const parts = [
|
|
988
|
-
`Studio RAG embedding request failed for provider ${config.
|
|
1055
|
+
`Studio RAG embedding request failed for ai.embedding (provider ${config.configuredProvider}, connection ${config.connection}, model ${config.model}).`,
|
|
989
1056
|
];
|
|
990
1057
|
if (config.baseUrl)
|
|
991
1058
|
parts.push(`Base URL: ${config.baseUrl}.`);
|
|
@@ -998,6 +1065,20 @@ function formatStudioRagOperationError(error, config) {
|
|
|
998
1065
|
parts.push(`Error: ${error.message}`);
|
|
999
1066
|
return new StudioHttpError(502, parts.join(" "));
|
|
1000
1067
|
}
|
|
1068
|
+
/** An existing pgvector table of another width: say where the new width comes from and how to resolve it, in config terms. */
|
|
1069
|
+
function studioRagWidthMismatchMessage(error, config) {
|
|
1070
|
+
const mismatch = `pgvector table "${error.table}" holds ${error.tableDimensions}-dimension vectors, but`;
|
|
1071
|
+
const rebuild = `Drop the table (DROP TABLE "${error.table}";) and build the index again`;
|
|
1072
|
+
const newTable = "point rag.storeConfig.pgvector.table at a new table";
|
|
1073
|
+
if (config.kind === "mock") {
|
|
1074
|
+
return `${mismatch} the mock embedder writes ${error.dimensions}. ${rebuild}, or ${newTable}.`;
|
|
1075
|
+
}
|
|
1076
|
+
const source = config.dimensions !== undefined
|
|
1077
|
+
? `ai.embedding.dimensions asks for ${error.dimensions}`
|
|
1078
|
+
: `embedding model ${config.model} returns ${error.dimensions}`;
|
|
1079
|
+
return (`${mismatch} ${source}. ${rebuild}, ${newTable}, ` +
|
|
1080
|
+
`or set ai.embedding.dimensions: ${error.tableDimensions} if model ${config.model} supports that width.`);
|
|
1081
|
+
}
|
|
1001
1082
|
function findApiCallError(error) {
|
|
1002
1083
|
const queue = [error];
|
|
1003
1084
|
for (let i = 0; i < queue.length; i++) {
|
|
@@ -1032,30 +1113,20 @@ function clearIncompatibleRagStore(state, config) {
|
|
|
1032
1113
|
return;
|
|
1033
1114
|
const embeddingsJsonPath = `${store.basePath}.embeddings.json`;
|
|
1034
1115
|
const embeddings = readOptionalJson(embeddingsJsonPath);
|
|
1035
|
-
if (embeddings?.dimensions === undefined
|
|
1116
|
+
if (embeddings?.dimensions === undefined)
|
|
1117
|
+
return;
|
|
1118
|
+
// The file store refuses mixed widths. When the width isn't known up front, vectors from a
|
|
1119
|
+
// different embedder may not match it, so they go too.
|
|
1120
|
+
const lockPath = join(state.schemaDir, "schema.lock.json");
|
|
1121
|
+
const compatible = config.dimensions !== undefined
|
|
1122
|
+
? embeddings.dimensions === config.dimensions
|
|
1123
|
+
: readOptionalJson(lockPath)?.embedderId === config.embedderId;
|
|
1124
|
+
if (compatible)
|
|
1036
1125
|
return;
|
|
1037
|
-
for (const path of [
|
|
1038
|
-
`${store.basePath}.embeddings.json`,
|
|
1039
|
-
`${store.basePath}.embeddings.bin`,
|
|
1040
|
-
join(state.schemaDir, "schema.lock.json"),
|
|
1041
|
-
]) {
|
|
1126
|
+
for (const path of [`${store.basePath}.embeddings.json`, `${store.basePath}.embeddings.bin`, lockPath]) {
|
|
1042
1127
|
rmSync(path, { force: true });
|
|
1043
1128
|
}
|
|
1044
1129
|
}
|
|
1045
|
-
function defaultEmbeddingDimensions(model) {
|
|
1046
|
-
if (model === "text-embedding-3-large")
|
|
1047
|
-
return 3072;
|
|
1048
|
-
return 1536;
|
|
1049
|
-
}
|
|
1050
|
-
function readPositiveIntegerEnv(value) {
|
|
1051
|
-
if (value === undefined || value.trim() === "")
|
|
1052
|
-
return undefined;
|
|
1053
|
-
const parsed = Number(value);
|
|
1054
|
-
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
1055
|
-
throw new StudioHttpError(400, `Invalid Studio RAG embedding dimensions: ${value}`);
|
|
1056
|
-
}
|
|
1057
|
-
return parsed;
|
|
1058
|
-
}
|
|
1059
1130
|
function serializeRagResult(result) {
|
|
1060
1131
|
return {
|
|
1061
1132
|
id: result.id,
|