@hiai-gg/docsmint 0.7.6 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -113
- package/dist/backend/index.js +860 -132
- package/dist/backend-account-runtime-cleanup.js +2 -0
- package/dist/backend-api-key-facade.js +2 -0
- package/dist/backend-pipeline-cancellation.js +2 -0
- package/dist/client.d.ts +4 -3
- package/dist/client.js +9 -6
- package/dist/frontend/{CategoryDialog-D_xFSWGh.js → CategoryDialog-B6zqtwE6.js} +2 -2
- package/dist/frontend/{ConfirmDialog-Cfq3D2Di.js → ConfirmDialog-Dk7wwcj9.js} +1 -1
- package/dist/frontend/{CreateSnapshotDialog-BHODxZWK.js → CreateSnapshotDialog-BPYjO1qN.js} +2 -2
- package/dist/frontend/{DatePicker-CZyCnQ1q.js → DatePicker-SiGLx9yh.js} +46 -46
- package/dist/frontend/{DeleteDialog-DaJOO6SB.js → DeleteDialog-D1jox8h1.js} +2 -2
- package/dist/frontend/{FolderNode-C_91NlEO.js → FolderNode-BG4l6I5C.js} +100 -99
- package/dist/frontend/{HiAiEditor-DtUGbQg6.js → HiAiEditor-BpmzvTCP.js} +977 -1008
- package/dist/frontend/{SettingsDialog-BlIZqk2-.js → SettingsDialog-2d6ZA4Ze.js} +3 -3
- package/dist/frontend/Sidebar-Db_ffhAb.js +2149 -0
- package/dist/frontend/{TagCreateDialog-BlGz4O9Z.js → TagCreateDialog-C5j3H49u.js} +2 -2
- package/dist/frontend/app-shell.js +3 -3
- package/dist/frontend/components/category-dialog.js +1 -1
- package/dist/frontend/components/create-snapshot-dialog.js +1 -1
- package/dist/frontend/components/delete-dialog.js +1 -1
- package/dist/frontend/components/editor/compact-editor.js +1 -1
- package/dist/frontend/components/editor/document-editor.js +1 -1
- package/dist/frontend/components/editor/extensions.js +1 -1
- package/dist/frontend/components/editor/markdown-toggle.js +1 -1
- package/dist/frontend/components/editor/markdown.js +1 -1
- package/dist/frontend/components/folder-node.js +1 -1
- package/dist/frontend/components/save-as-dialog.js +2 -2
- package/dist/frontend/components/settings.js +1 -1
- package/dist/frontend/components/sidebar.js +1 -1
- package/dist/frontend/components/tag-create-dialog.js +1 -1
- package/dist/frontend/components/version-history.js +3 -3
- package/dist/frontend/dashboard.js +1034 -1034
- package/dist/frontend/{dialog-tHAH3-9M.js → dialog-DC97Hnxo.js} +1 -1
- package/dist/frontend/dist-BLpDfrDC.js +316 -0
- package/dist/frontend/dist-D7zL6zdU.js +40 -0
- package/dist/frontend/{dist-Cg1BpNS7.js → dist-qd_OnyUC.js} +492 -516
- package/dist/frontend/dropdown-menu-COZwS7c0.js +493 -0
- package/dist/frontend/editorExtensions-Cu__YKW7.js +158 -0
- package/dist/frontend/ellipsis-vertical-CObXWXZW.js +41 -0
- package/dist/frontend/frontend.css +1 -1
- package/dist/frontend/{editorExtensions-B-mKiWqq.js → lib-DYgslUw1.js} +6782 -6935
- package/dist/frontend/loader-circle-DnyUzAmN.js +25 -0
- package/dist/frontend/{markdown-CMij27Gg.js → markdown-Co8ZAKf4.js} +2 -1
- package/dist/frontend/search.js +2 -2
- package/dist/frontend/{confirm-dialog-BFTxjLv-.js → share-context-DooXPYKd.js} +2 -3
- package/dist/frontend/theme.js +1 -1
- package/dist/frontend/{theme.svelte-Bzitjfh9.js → theme.svelte-gLPpG6eU.js} +1 -0
- package/dist/frontend-ssr/app-shell.js +3 -3
- package/dist/frontend-ssr/assets/{FolderNode-CClgk8BL.js → FolderNode-CscMQ9sf.js} +2 -1
- package/dist/frontend-ssr/assets/{HiAiEditor-_-m7MJDd.js → HiAiEditor-DCpulE6N.js} +4 -50
- package/dist/frontend-ssr/assets/{SettingsDialog-BdvHRAxF.js → SettingsDialog-CpPxZahd.js} +1 -1
- package/dist/frontend-ssr/assets/{Sidebar-B3Saqn0O.js → Sidebar-BJ1XZ1DH.js} +7 -5
- package/dist/frontend-ssr/assets/dist-CshZmNmp.js +50 -0
- package/dist/frontend-ssr/assets/dist-DqeAICTy.js +300 -0
- package/dist/frontend-ssr/assets/{dropdown-menu-CdEpn_LT.js → dropdown-menu--acO7nXs.js} +1 -94
- package/dist/frontend-ssr/assets/ellipsis-vertical-BErUuhWg.js +96 -0
- package/dist/frontend-ssr/assets/{markdown-MhPwNWrC.js → markdown-Co31-Tl2.js} +1 -1
- package/dist/frontend-ssr/assets/{confirm-dialog-DeOR0SnQ.js → share-context-BvXQrIIH.js} +0 -1
- package/dist/frontend-ssr/assets/{theme.svelte-shZrSxz8.js → theme.svelte-CGgnJc3k.js} +1 -0
- package/dist/frontend-ssr/components/editor/compact-editor.js +1 -1
- package/dist/frontend-ssr/components/editor/document-editor.js +1 -1
- package/dist/frontend-ssr/components/editor/extensions.js +1 -1
- package/dist/frontend-ssr/components/editor/markdown-toggle.js +1 -1
- package/dist/frontend-ssr/components/editor/markdown.js +1 -1
- package/dist/frontend-ssr/components/folder-node.js +1 -1
- package/dist/frontend-ssr/components/settings.js +1 -1
- package/dist/frontend-ssr/components/sidebar.js +1 -1
- package/dist/frontend-ssr/dashboard.js +6 -4
- package/dist/frontend-ssr/search.js +1 -1
- package/dist/frontend-ssr/theme.js +1 -1
- package/dist/lifecycle-runtime.js +42 -4
- package/dist/mcp-cli.js +9 -6
- package/dist/mcp-server.js +9 -6
- package/dist/types.d.ts +12 -0
- package/package.json +1 -1
- package/packages/cli/src/index.ts +1 -1
- package/packages/db/src/schema.ts +2 -0
- package/server.json +2 -2
- package/skills/docsmint-document-manager/SKILL.md +2 -2
- package/dist/frontend/Sidebar-BVGm5BRR.js +0 -2147
- package/dist/frontend/dropdown-menu-AJYeFPHu.js +0 -530
- package/dist/frontend-ssr/assets/{DatePicker-TZVeMZ2S.js → DatePicker-CCylVH0q.js} +12 -12
- package/dist/frontend-ssr/assets/{editorExtensions-qNlZVP98.js → editorExtensions-rhletNji.js} +1 -1
package/dist/backend/index.js
CHANGED
|
@@ -45791,6 +45791,8 @@ var init_schema2 = __esm(() => {
|
|
|
45791
45791
|
completedBatches: integer("completed_batches").notNull().default(0),
|
|
45792
45792
|
failedBatches: integer("failed_batches").notNull().default(0),
|
|
45793
45793
|
errorCode: text("error_code"),
|
|
45794
|
+
graphErrorCode: text("graph_error_code"),
|
|
45795
|
+
summarizeErrorCode: text("summarize_error_code"),
|
|
45794
45796
|
attempts: integer("attempts").notNull().default(0),
|
|
45795
45797
|
requestedAt: timestamp("requested_at").defaultNow().notNull(),
|
|
45796
45798
|
startedAt: timestamp("started_at"),
|
|
@@ -79815,10 +79817,26 @@ var init_kysely_adapter = __esm(() => {
|
|
|
79815
79817
|
init_dist3();
|
|
79816
79818
|
});
|
|
79817
79819
|
|
|
79820
|
+
// ../../backend/src/lib/openrouter-public-matrix.ts
|
|
79821
|
+
var OPENROUTER_PUBLIC_BASE_URL = "https://openrouter.ai/api/v1", OPENROUTER_PUBLIC_EXPANSION_MODELS, OPENROUTER_PUBLIC_RERANK_MODELS;
|
|
79822
|
+
var init_openrouter_public_matrix = __esm(() => {
|
|
79823
|
+
OPENROUTER_PUBLIC_EXPANSION_MODELS = {
|
|
79824
|
+
primary: "poolside/laguna-xs-2.1",
|
|
79825
|
+
fallback: "openai/gpt-5.6-luna",
|
|
79826
|
+
fallback_2: "google/gemini-3.5-flash-lite"
|
|
79827
|
+
};
|
|
79828
|
+
OPENROUTER_PUBLIC_RERANK_MODELS = {
|
|
79829
|
+
primary: "voyageai/rerank-2.5",
|
|
79830
|
+
fallback: "cohere/rerank-v3.5",
|
|
79831
|
+
fallback_2: "nvidia/llama-nemotron-rerank-vl-1b-v2:free"
|
|
79832
|
+
};
|
|
79833
|
+
});
|
|
79834
|
+
|
|
79818
79835
|
// ../../backend/src/lib/config-schema.ts
|
|
79819
79836
|
var adminApiKeyPlaceholder, envSchema;
|
|
79820
79837
|
var init_config_schema = __esm(() => {
|
|
79821
79838
|
init_zod();
|
|
79839
|
+
init_openrouter_public_matrix();
|
|
79822
79840
|
adminApiKeyPlaceholder = /^(?:change[-_ ]?me|changeme|your[-_ ]?api[-_ ]?key)(?:\b|[-_ ])/i;
|
|
79823
79841
|
envSchema = exports_external2.object({
|
|
79824
79842
|
DATABASE_URL: exports_external2.string().default("postgresql://hiai_app:changeme@localhost:5437/hiai_docs"),
|
|
@@ -79875,6 +79893,9 @@ var init_config_schema = __esm(() => {
|
|
|
79875
79893
|
EMBEDDING_FALLBACK_BASE_URL: exports_external2.string().optional(),
|
|
79876
79894
|
EMBEDDING_FALLBACK_API_KEY: exports_external2.string().optional(),
|
|
79877
79895
|
EMBEDDING_FALLBACK_MODEL: exports_external2.string().optional(),
|
|
79896
|
+
EMBEDDING_FALLBACK_2_BASE_URL: exports_external2.string().optional(),
|
|
79897
|
+
EMBEDDING_FALLBACK_2_API_KEY: exports_external2.string().optional(),
|
|
79898
|
+
EMBEDDING_FALLBACK_2_MODEL: exports_external2.string().optional(),
|
|
79878
79899
|
API_PORT: exports_external2.coerce.number().default(50700),
|
|
79879
79900
|
NODE_ENV: exports_external2.enum(["development", "production", "test"]).default("development"),
|
|
79880
79901
|
LOG_LEVEL: exports_external2.enum(["trace", "debug", "info", "warn", "error", "fatal"]).default("info"),
|
|
@@ -79901,14 +79922,34 @@ var init_config_schema = __esm(() => {
|
|
|
79901
79922
|
GRAPH_EXTRACT_FALLBACK_BASE_URL: exports_external2.string().optional(),
|
|
79902
79923
|
GRAPH_EXTRACT_FALLBACK_API_KEY: exports_external2.string().optional(),
|
|
79903
79924
|
GRAPH_EXTRACT_FALLBACK_MODEL: exports_external2.string().optional(),
|
|
79925
|
+
GRAPH_EXTRACT_FALLBACK_2_BASE_URL: exports_external2.string().optional(),
|
|
79926
|
+
GRAPH_EXTRACT_FALLBACK_2_API_KEY: exports_external2.string().optional(),
|
|
79927
|
+
GRAPH_EXTRACT_FALLBACK_2_MODEL: exports_external2.string().optional(),
|
|
79904
79928
|
GRAPH_EXTRACT_MIN_CONFIDENCE: exports_external2.coerce.number().min(0).max(1).default(0.5),
|
|
79905
79929
|
SEARCH_EXPANSION_ENABLED: exports_external2.string().optional().default("true").transform((v2) => v2 === "true"),
|
|
79906
|
-
SEARCH_EXPANSION_BASE_URL: exports_external2.string().default(
|
|
79930
|
+
SEARCH_EXPANSION_BASE_URL: exports_external2.string().default(OPENROUTER_PUBLIC_BASE_URL),
|
|
79907
79931
|
SEARCH_EXPANSION_API_KEY: exports_external2.string().optional(),
|
|
79908
|
-
SEARCH_EXPANSION_MODEL: exports_external2.string().default(
|
|
79909
|
-
SEARCH_EXPANSION_FALLBACK_BASE_URL: exports_external2.string().default(
|
|
79932
|
+
SEARCH_EXPANSION_MODEL: exports_external2.string().default(OPENROUTER_PUBLIC_EXPANSION_MODELS.primary),
|
|
79933
|
+
SEARCH_EXPANSION_FALLBACK_BASE_URL: exports_external2.string().default(OPENROUTER_PUBLIC_BASE_URL),
|
|
79910
79934
|
SEARCH_EXPANSION_FALLBACK_API_KEY: exports_external2.string().optional(),
|
|
79911
|
-
SEARCH_EXPANSION_FALLBACK_MODEL: exports_external2.string().default(
|
|
79935
|
+
SEARCH_EXPANSION_FALLBACK_MODEL: exports_external2.string().default(OPENROUTER_PUBLIC_EXPANSION_MODELS.fallback),
|
|
79936
|
+
SEARCH_EXPANSION_FALLBACK_2_BASE_URL: exports_external2.string().default(OPENROUTER_PUBLIC_BASE_URL),
|
|
79937
|
+
SEARCH_EXPANSION_FALLBACK_2_API_KEY: exports_external2.string().optional(),
|
|
79938
|
+
SEARCH_EXPANSION_FALLBACK_2_MODEL: exports_external2.string().default(OPENROUTER_PUBLIC_EXPANSION_MODELS.fallback_2),
|
|
79939
|
+
SEARCH_RERANK_ENABLED: exports_external2.string().optional().default("true").transform((v2) => v2 === "true"),
|
|
79940
|
+
SEARCH_RERANK_BASE_URL: exports_external2.string().default(OPENROUTER_PUBLIC_BASE_URL),
|
|
79941
|
+
SEARCH_RERANK_API_KEY: exports_external2.string().optional(),
|
|
79942
|
+
SEARCH_RERANK_MODEL: exports_external2.string().default(OPENROUTER_PUBLIC_RERANK_MODELS.primary),
|
|
79943
|
+
SEARCH_RERANK_FALLBACK_BASE_URL: exports_external2.string().default(OPENROUTER_PUBLIC_BASE_URL),
|
|
79944
|
+
SEARCH_RERANK_FALLBACK_API_KEY: exports_external2.string().optional(),
|
|
79945
|
+
SEARCH_RERANK_FALLBACK_MODEL: exports_external2.string().default(OPENROUTER_PUBLIC_RERANK_MODELS.fallback),
|
|
79946
|
+
SEARCH_RERANK_FALLBACK_2_BASE_URL: exports_external2.string().default(OPENROUTER_PUBLIC_BASE_URL),
|
|
79947
|
+
SEARCH_RERANK_FALLBACK_2_API_KEY: exports_external2.string().optional(),
|
|
79948
|
+
SEARCH_RERANK_FALLBACK_2_MODEL: exports_external2.string().default(OPENROUTER_PUBLIC_RERANK_MODELS.fallback_2),
|
|
79949
|
+
SEARCH_RERANK_TIMEOUT_MS: exports_external2.coerce.number().int().min(250).max(30000).default(3000),
|
|
79950
|
+
SEARCH_RERANK_TOP_N: exports_external2.coerce.number().int().min(1).max(100).default(20),
|
|
79951
|
+
SEARCH_RERANK_MAX_CHARS: exports_external2.coerce.number().int().min(120).max(8000).default(1500),
|
|
79952
|
+
SEARCH_RERANK_GRAPH_POSITION: exports_external2.enum(["before", "after"]).default("after"),
|
|
79912
79953
|
SEARCH_EXPANSION_TIMEOUT_MS: exports_external2.coerce.number().int().min(1000).max(300000).default(6000),
|
|
79913
79954
|
SEARCH_VECTOR_PROVIDER_TIMEOUT_MS: exports_external2.coerce.number().int().min(250).max(30000).default(2500),
|
|
79914
79955
|
SEARCH_EXPANSION_CACHE_TTL_SECONDS: exports_external2.coerce.number().int().min(0).max(2592000).default(86400),
|
|
@@ -195670,6 +195711,10 @@ var METRIC_NAMES = {
|
|
|
195670
195711
|
SEARCH_EXPANSION_LOW_VECTOR_SIMILARITY_TOTAL: "search_expansion_reason_low_vector_similarity_total",
|
|
195671
195712
|
SEARCH_EXPANSION_LANGUAGE_MISMATCH_TOTAL: "search_expansion_reason_language_mismatch_total",
|
|
195672
195713
|
SEARCH_EXPANSION_EMPTY_CANDIDATES_TOTAL: "search_expansion_reason_empty_candidates_total",
|
|
195714
|
+
SEARCH_RERANK_DURATION_MS: "search_rerank_duration_ms",
|
|
195715
|
+
SEARCH_RERANK_SUCCESS_TOTAL: "search_rerank_success_total",
|
|
195716
|
+
SEARCH_RERANK_FALLBACK_TOTAL: "search_rerank_fallback_total",
|
|
195717
|
+
SEARCH_RERANK_CANDIDATES_TOTAL: "search_rerank_candidates_total",
|
|
195673
195718
|
SEARCH_EXACT_DURATION_MS: "search_exact_duration_ms",
|
|
195674
195719
|
SEARCH_EXACT_ERRORS_TOTAL: "search_exact_errors_total",
|
|
195675
195720
|
SEARCH_EXACT_CANDIDATES_TOTAL: "search_exact_candidates_total",
|
|
@@ -195841,7 +195886,7 @@ function recordSearchExpansionMetrics(input) {
|
|
|
195841
195886
|
break;
|
|
195842
195887
|
}
|
|
195843
195888
|
}
|
|
195844
|
-
if (input.used !== false && input.model && input.model === input.fallbackModel) {
|
|
195889
|
+
if (input.used !== false && input.model && (input.model === input.fallbackModel || input.model === input.fallback2Model)) {
|
|
195845
195890
|
incrementCounter(METRIC_NAMES.SEARCH_EXPANSION_FALLBACK_TOTAL);
|
|
195846
195891
|
} else if (input.used !== false && input.model) {
|
|
195847
195892
|
incrementCounter(METRIC_NAMES.SEARCH_EXPANSION_PRIMARY_TOTAL);
|
|
@@ -196176,31 +196221,52 @@ async function getEmbedding(text3) {
|
|
|
196176
196221
|
recordDuration(METRIC_NAMES.EMBEDDING_DURATION_MS, Date.now() - start);
|
|
196177
196222
|
}
|
|
196178
196223
|
}
|
|
196179
|
-
|
|
196180
|
-
const
|
|
196181
|
-
|
|
196182
|
-
providers.push({
|
|
196224
|
+
function configuredEmbeddingProviders() {
|
|
196225
|
+
const slots = [
|
|
196226
|
+
{
|
|
196183
196227
|
baseUrl: config3.EMBEDDING_BASE_URL,
|
|
196184
196228
|
apiKey: config3.EMBEDDING_API_KEY,
|
|
196185
196229
|
model: config3.EMBEDDING_MODEL,
|
|
196186
196230
|
provider: "primary"
|
|
196187
|
-
}
|
|
196188
|
-
|
|
196189
|
-
if (config3.EMBEDDING_FALLBACK_BASE_URL && config3.EMBEDDING_FALLBACK_MODEL) {
|
|
196190
|
-
providers.push({
|
|
196231
|
+
},
|
|
196232
|
+
{
|
|
196191
196233
|
baseUrl: config3.EMBEDDING_FALLBACK_BASE_URL,
|
|
196192
196234
|
apiKey: config3.EMBEDDING_FALLBACK_API_KEY,
|
|
196193
196235
|
model: config3.EMBEDDING_FALLBACK_MODEL,
|
|
196194
196236
|
provider: "fallback"
|
|
196195
|
-
}
|
|
196196
|
-
|
|
196237
|
+
},
|
|
196238
|
+
{
|
|
196239
|
+
baseUrl: config3.EMBEDDING_FALLBACK_2_BASE_URL,
|
|
196240
|
+
apiKey: config3.EMBEDDING_FALLBACK_2_API_KEY,
|
|
196241
|
+
model: config3.EMBEDDING_FALLBACK_2_MODEL,
|
|
196242
|
+
provider: "fallback_2"
|
|
196243
|
+
}
|
|
196244
|
+
];
|
|
196245
|
+
return slots.flatMap((slot) => slot.baseUrl && slot.model ? [
|
|
196246
|
+
{
|
|
196247
|
+
baseUrl: slot.baseUrl,
|
|
196248
|
+
apiKey: slot.apiKey,
|
|
196249
|
+
model: slot.model,
|
|
196250
|
+
provider: slot.provider
|
|
196251
|
+
}
|
|
196252
|
+
] : []);
|
|
196253
|
+
}
|
|
196254
|
+
function assignProviderError(provider, message2, errors6) {
|
|
196255
|
+
if (provider === "primary")
|
|
196256
|
+
errors6.primaryError = message2;
|
|
196257
|
+
else if (provider === "fallback")
|
|
196258
|
+
errors6.fallbackError = message2;
|
|
196259
|
+
else
|
|
196260
|
+
errors6.fallback2Error = message2;
|
|
196261
|
+
}
|
|
196262
|
+
async function getEmbeddingInner(text3) {
|
|
196263
|
+
const providers = configuredEmbeddingProviders();
|
|
196197
196264
|
if (providers.length === 0) {
|
|
196198
196265
|
logger3.warn("No embedding provider configured (primary or fallback); embedding result is unavailable");
|
|
196199
196266
|
incrementCounter(METRIC_NAMES.EMBEDDING_ZERO);
|
|
196200
196267
|
return { ok: false, code: "not_configured" };
|
|
196201
196268
|
}
|
|
196202
|
-
|
|
196203
|
-
let fallbackError;
|
|
196269
|
+
const errors6 = {};
|
|
196204
196270
|
let finalCode = "provider_error";
|
|
196205
196271
|
for (const provider of providers) {
|
|
196206
196272
|
try {
|
|
@@ -196208,11 +196274,7 @@ async function getEmbeddingInner(text3) {
|
|
|
196208
196274
|
const validation = validateEmbeddingVector(vector5);
|
|
196209
196275
|
if (!validation.ok) {
|
|
196210
196276
|
finalCode = validation.code;
|
|
196211
|
-
|
|
196212
|
-
if (provider.provider === "primary")
|
|
196213
|
-
primaryError = message2;
|
|
196214
|
-
else
|
|
196215
|
-
fallbackError = message2;
|
|
196277
|
+
assignProviderError(provider.provider, `provider returned ${validation.code}`, errors6);
|
|
196216
196278
|
logger3.warn({
|
|
196217
196279
|
model: provider.model,
|
|
196218
196280
|
provider: provider.provider,
|
|
@@ -196235,11 +196297,7 @@ async function getEmbeddingInner(text3) {
|
|
|
196235
196297
|
};
|
|
196236
196298
|
} catch (err) {
|
|
196237
196299
|
finalCode = failureCodeFromError(err);
|
|
196238
|
-
|
|
196239
|
-
if (provider.provider === "primary")
|
|
196240
|
-
primaryError = message2;
|
|
196241
|
-
else
|
|
196242
|
-
fallbackError = message2;
|
|
196300
|
+
assignProviderError(provider.provider, safeErrorMessage(err), errors6);
|
|
196243
196301
|
logger3.warn({ err, model: provider.model, provider: provider.provider }, "Embedding provider failed, trying next configured provider");
|
|
196244
196302
|
}
|
|
196245
196303
|
}
|
|
@@ -196247,8 +196305,7 @@ async function getEmbeddingInner(text3) {
|
|
|
196247
196305
|
return {
|
|
196248
196306
|
ok: false,
|
|
196249
196307
|
code: finalCode,
|
|
196250
|
-
...
|
|
196251
|
-
...fallbackError ? { fallbackError } : {}
|
|
196308
|
+
...errors6
|
|
196252
196309
|
};
|
|
196253
196310
|
}
|
|
196254
196311
|
function safeErrorMessage(error53) {
|
|
@@ -196359,7 +196416,8 @@ var basePipelineJobFields = {
|
|
|
196359
196416
|
requestedAt: exports_external2.iso.datetime(),
|
|
196360
196417
|
source: pipelineSourceSchema,
|
|
196361
196418
|
refreshMode: pipelineRefreshModeSchema,
|
|
196362
|
-
embeddingContextHash: exports_external2.string().min(1).optional()
|
|
196419
|
+
embeddingContextHash: exports_external2.string().min(1).optional(),
|
|
196420
|
+
warningRetry: exports_external2.literal(true).optional()
|
|
196363
196421
|
};
|
|
196364
196422
|
function normalizeLegacyPipelineJob(value) {
|
|
196365
196423
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
@@ -228592,6 +228650,33 @@ function encodeS3CopySource(bucket, key) {
|
|
|
228592
228650
|
// ../../backend/src/api/routes/documents.ts
|
|
228593
228651
|
init_logger3();
|
|
228594
228652
|
|
|
228653
|
+
// ../../backend/src/lib/pipeline-error.ts
|
|
228654
|
+
class PipelineProviderError extends Error {
|
|
228655
|
+
code;
|
|
228656
|
+
retryable;
|
|
228657
|
+
constructor(code2, retryable, message2 = code2) {
|
|
228658
|
+
super(message2);
|
|
228659
|
+
this.code = code2;
|
|
228660
|
+
this.retryable = retryable;
|
|
228661
|
+
this.name = "PipelineProviderError";
|
|
228662
|
+
}
|
|
228663
|
+
}
|
|
228664
|
+
function pipelineErrorCode(error53, fallback) {
|
|
228665
|
+
if (error53 instanceof PipelineProviderError)
|
|
228666
|
+
return error53.code;
|
|
228667
|
+
return fallback;
|
|
228668
|
+
}
|
|
228669
|
+
function isRetryablePipelineError(code2) {
|
|
228670
|
+
return new Set([
|
|
228671
|
+
"provider_failure",
|
|
228672
|
+
"provider_timeout",
|
|
228673
|
+
"invalid_provider_response",
|
|
228674
|
+
"queue_enqueue_failed",
|
|
228675
|
+
"age_unavailable",
|
|
228676
|
+
"age_persist_failed"
|
|
228677
|
+
]).has(code2);
|
|
228678
|
+
}
|
|
228679
|
+
|
|
228595
228680
|
// ../../backend/src/lib/version-prune.ts
|
|
228596
228681
|
init_schema2();
|
|
228597
228682
|
init_drizzle_orm();
|
|
@@ -228627,6 +228712,30 @@ async function maybePruneVersions(documentId) {
|
|
|
228627
228712
|
}
|
|
228628
228713
|
|
|
228629
228714
|
// ../../backend/src/api/routes/documents.ts
|
|
228715
|
+
init_names();
|
|
228716
|
+
|
|
228717
|
+
// ../../backend/src/queue/pipeline-warning-retry.ts
|
|
228718
|
+
function resolveActiveWarningRetry(stageStarted, retryJobExists) {
|
|
228719
|
+
if (!stageStarted)
|
|
228720
|
+
return "enqueue";
|
|
228721
|
+
return retryJobExists ? "deduplicate" : "conflict";
|
|
228722
|
+
}
|
|
228723
|
+
function warningRetryJobId(stage, generationId) {
|
|
228724
|
+
return `${stage}-warning-retry-${generationId}`;
|
|
228725
|
+
}
|
|
228726
|
+
function summaryJobIdForPipeline(job2) {
|
|
228727
|
+
return job2.warningRetry ? warningRetryJobId("summarize", job2.generationId) : JOB_IDS.summarize(job2.generationId, job2.workspaceId);
|
|
228728
|
+
}
|
|
228729
|
+
function planWarningStageRetry(state) {
|
|
228730
|
+
const graph = state.graphStatus === "failed" && isRetryablePipelineError(state.graphErrorCode ?? "graph_failed");
|
|
228731
|
+
const summarize = state.summarizeStatus === "failed" && isRetryablePipelineError(state.summarizeErrorCode ?? "summary_failed");
|
|
228732
|
+
if (!graph && !summarize)
|
|
228733
|
+
return null;
|
|
228734
|
+
return { graph, summarize, entryStage: graph ? "graph" : "summarize" };
|
|
228735
|
+
}
|
|
228736
|
+
|
|
228737
|
+
// ../../backend/src/api/routes/documents.ts
|
|
228738
|
+
init_queues();
|
|
228630
228739
|
class DocumentPurgeNotFoundError extends Error {
|
|
228631
228740
|
constructor() {
|
|
228632
228741
|
super("document_purge_not_found");
|
|
@@ -229182,6 +229291,8 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
229182
229291
|
embedStatus: documentPipelineRuns.embedStatus,
|
|
229183
229292
|
graphStatus: documentPipelineRuns.graphStatus,
|
|
229184
229293
|
summarizeStatus: documentPipelineRuns.summarizeStatus,
|
|
229294
|
+
graphErrorCode: documentPipelineRuns.graphErrorCode,
|
|
229295
|
+
summarizeErrorCode: documentPipelineRuns.summarizeErrorCode,
|
|
229185
229296
|
finalizeStatus: documentPipelineRuns.finalizeStatus,
|
|
229186
229297
|
totalBatches: documentPipelineRuns.totalBatches,
|
|
229187
229298
|
completedBatches: documentPipelineRuns.completedBatches,
|
|
@@ -229211,6 +229322,22 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
229211
229322
|
completed: run3.completedBatches,
|
|
229212
229323
|
failed: run3.failedBatches
|
|
229213
229324
|
},
|
|
229325
|
+
warnings: [
|
|
229326
|
+
...run3.graphStatus === "failed" ? [
|
|
229327
|
+
{
|
|
229328
|
+
stage: "graph",
|
|
229329
|
+
code: run3.graphErrorCode ?? "graph_failed",
|
|
229330
|
+
retryable: isRetryablePipelineError(run3.graphErrorCode ?? "graph_failed")
|
|
229331
|
+
}
|
|
229332
|
+
] : [],
|
|
229333
|
+
...run3.summarizeStatus === "failed" ? [
|
|
229334
|
+
{
|
|
229335
|
+
stage: "summarize",
|
|
229336
|
+
code: run3.summarizeErrorCode ?? "summary_failed",
|
|
229337
|
+
retryable: isRetryablePipelineError(run3.summarizeErrorCode ?? "summary_failed")
|
|
229338
|
+
}
|
|
229339
|
+
] : []
|
|
229340
|
+
],
|
|
229214
229341
|
updatedAt: run3.updatedAt
|
|
229215
229342
|
};
|
|
229216
229343
|
} catch (err) {
|
|
@@ -229218,6 +229345,194 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
229218
229345
|
set2.status = 500;
|
|
229219
229346
|
return { error: "Failed to load pipeline progress" };
|
|
229220
229347
|
}
|
|
229348
|
+
}).post("/documents/:id/pipeline/retry-warnings", async ({ params: params2, set: set2, request }) => {
|
|
229349
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
229350
|
+
if (!parsedParams.success) {
|
|
229351
|
+
set2.status = 400;
|
|
229352
|
+
return { error: "Invalid document id" };
|
|
229353
|
+
}
|
|
229354
|
+
const documentId = parsedParams.data.id;
|
|
229355
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
229356
|
+
const rl = await writeRateLimiter(ip, request);
|
|
229357
|
+
if (!rl.allowed) {
|
|
229358
|
+
set2.status = 429;
|
|
229359
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
229360
|
+
return { error: "Too many requests" };
|
|
229361
|
+
}
|
|
229362
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
229363
|
+
const access = await resolveContentAccess(request);
|
|
229364
|
+
const ctx = access.ctx;
|
|
229365
|
+
if (ctx.role === "none") {
|
|
229366
|
+
set2.status = 401;
|
|
229367
|
+
return { error: "Unauthorized" };
|
|
229368
|
+
}
|
|
229369
|
+
if (!canAccessContent(access, "write")) {
|
|
229370
|
+
set2.status = 403;
|
|
229371
|
+
return { error: "Forbidden" };
|
|
229372
|
+
}
|
|
229373
|
+
let claimed;
|
|
229374
|
+
try {
|
|
229375
|
+
const retry2 = await withTenant(ctx, async (tx) => {
|
|
229376
|
+
await acquireDocumentPipelineLock(tx, documentId);
|
|
229377
|
+
const [run3] = await tx.select({
|
|
229378
|
+
status: documentPipelineRuns.status,
|
|
229379
|
+
generationId: documentPipelineRuns.generationId,
|
|
229380
|
+
revision: documentPipelineRuns.revision,
|
|
229381
|
+
ownerId: documentPipelineRuns.ownerId,
|
|
229382
|
+
workspaceId: documentPipelineRuns.workspaceId,
|
|
229383
|
+
refreshMode: documentPipelineRuns.refreshMode,
|
|
229384
|
+
embeddingContextHash: documentPipelineRuns.embeddingContextHash,
|
|
229385
|
+
embedStatus: documentPipelineRuns.embedStatus,
|
|
229386
|
+
graphStatus: documentPipelineRuns.graphStatus,
|
|
229387
|
+
summarizeStatus: documentPipelineRuns.summarizeStatus,
|
|
229388
|
+
graphErrorCode: documentPipelineRuns.graphErrorCode,
|
|
229389
|
+
summarizeErrorCode: documentPipelineRuns.summarizeErrorCode
|
|
229390
|
+
}).from(documentPipelineRuns).innerJoin(documents, and(eq(documents.id, documentPipelineRuns.documentId), eq(documents.activeEmbeddingGeneration, documentPipelineRuns.generationId), eq(documents.contentHash, documentPipelineRuns.revision))).where(and(eq(documentPipelineRuns.documentId, documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), eq(documentPipelineRuns.embedStatus, "ready"), isNull(documents.deletedAt), ...access.restricted ? [
|
|
229391
|
+
access.categoryId ? effectiveDocumentCategoryCondition(documents.categoryId, documents.folderId, ctx, access.categoryId) : sql`false`
|
|
229392
|
+
] : [])).orderBy(desc(documentPipelineRuns.updatedAt)).limit(1).for("update");
|
|
229393
|
+
if (!run3)
|
|
229394
|
+
return null;
|
|
229395
|
+
if (run3.status === "processing") {
|
|
229396
|
+
const graph2 = ["retrying", "processing"].includes(run3.graphStatus);
|
|
229397
|
+
const summarize2 = ["retrying", "processing"].includes(run3.summarizeStatus);
|
|
229398
|
+
return graph2 || summarize2 ? {
|
|
229399
|
+
...run3,
|
|
229400
|
+
graph: graph2,
|
|
229401
|
+
summarize: summarize2,
|
|
229402
|
+
deduplicated: true,
|
|
229403
|
+
stageStarted: run3.graphStatus === "processing" || run3.summarizeStatus === "processing"
|
|
229404
|
+
} : null;
|
|
229405
|
+
}
|
|
229406
|
+
if (run3.status !== "ready_with_warnings")
|
|
229407
|
+
return null;
|
|
229408
|
+
const plan = planWarningStageRetry(run3);
|
|
229409
|
+
if (!plan)
|
|
229410
|
+
return null;
|
|
229411
|
+
const { graph, summarize } = plan;
|
|
229412
|
+
await tx.update(documentPipelineRuns).set({
|
|
229413
|
+
status: "processing",
|
|
229414
|
+
finalizeStatus: "pending",
|
|
229415
|
+
...graph ? { graphStatus: "retrying", graphErrorCode: null } : {},
|
|
229416
|
+
...summarize ? {
|
|
229417
|
+
summarizeStatus: "retrying",
|
|
229418
|
+
summarizeErrorCode: null
|
|
229419
|
+
} : {},
|
|
229420
|
+
updatedAt: new Date
|
|
229421
|
+
}).where(eq(documentPipelineRuns.generationId, run3.generationId));
|
|
229422
|
+
return {
|
|
229423
|
+
...run3,
|
|
229424
|
+
graph,
|
|
229425
|
+
summarize,
|
|
229426
|
+
deduplicated: false,
|
|
229427
|
+
stageStarted: false
|
|
229428
|
+
};
|
|
229429
|
+
});
|
|
229430
|
+
if (!retry2) {
|
|
229431
|
+
set2.status = 409;
|
|
229432
|
+
return { error: "No retryable warning stages" };
|
|
229433
|
+
}
|
|
229434
|
+
if (!retry2.deduplicated)
|
|
229435
|
+
claimed = retry2;
|
|
229436
|
+
const stage = retry2.graph ? "graph" : "summarize";
|
|
229437
|
+
const retryJobId = warningRetryJobId(stage, retry2.generationId);
|
|
229438
|
+
const staleJobs = [
|
|
229439
|
+
...retry2.graph ? [
|
|
229440
|
+
[
|
|
229441
|
+
"graph",
|
|
229442
|
+
JOB_IDS.graph(retry2.generationId, retry2.workspaceId ?? undefined)
|
|
229443
|
+
]
|
|
229444
|
+
] : [],
|
|
229445
|
+
...retry2.summarize ? [
|
|
229446
|
+
[
|
|
229447
|
+
"summarize",
|
|
229448
|
+
JOB_IDS.summarize(retry2.generationId, retry2.workspaceId ?? undefined)
|
|
229449
|
+
]
|
|
229450
|
+
] : [],
|
|
229451
|
+
[
|
|
229452
|
+
"finalize",
|
|
229453
|
+
JOB_IDS.finalize(retry2.generationId, retry2.workspaceId ?? undefined)
|
|
229454
|
+
]
|
|
229455
|
+
];
|
|
229456
|
+
if (!retry2.deduplicated) {
|
|
229457
|
+
for (const [queuedStage, jobId] of staleJobs) {
|
|
229458
|
+
const oldJob = await getPipelineQueue(queuedStage, config3.REDIS_URL).getJob(jobId);
|
|
229459
|
+
await oldJob?.remove();
|
|
229460
|
+
}
|
|
229461
|
+
}
|
|
229462
|
+
const queue2 = getPipelineQueue(stage, config3.REDIS_URL);
|
|
229463
|
+
const existingRetryJob = await queue2.getJob(retryJobId);
|
|
229464
|
+
if (retry2.deduplicated) {
|
|
229465
|
+
const activeRetry = resolveActiveWarningRetry(retry2.stageStarted, Boolean(existingRetryJob));
|
|
229466
|
+
if (activeRetry === "conflict") {
|
|
229467
|
+
set2.status = 409;
|
|
229468
|
+
return { error: "No retryable warning stages" };
|
|
229469
|
+
}
|
|
229470
|
+
if (activeRetry === "deduplicate")
|
|
229471
|
+
return {
|
|
229472
|
+
documentId,
|
|
229473
|
+
generationId: retry2.generationId,
|
|
229474
|
+
deduplicated: true,
|
|
229475
|
+
retriedStages: [
|
|
229476
|
+
...retry2.graph ? ["graph"] : [],
|
|
229477
|
+
...retry2.summarize ? ["summarize"] : []
|
|
229478
|
+
]
|
|
229479
|
+
};
|
|
229480
|
+
}
|
|
229481
|
+
const existingRetryState = await existingRetryJob?.getState();
|
|
229482
|
+
if (existingRetryState === "completed" || existingRetryState === "failed") {
|
|
229483
|
+
await existingRetryJob?.remove();
|
|
229484
|
+
}
|
|
229485
|
+
const data = {
|
|
229486
|
+
schemaVersion: PIPELINE_SCHEMA_VERSION,
|
|
229487
|
+
stage,
|
|
229488
|
+
documentId,
|
|
229489
|
+
ownerId: retry2.ownerId,
|
|
229490
|
+
...retry2.workspaceId ? { workspaceId: retry2.workspaceId } : {},
|
|
229491
|
+
generationId: retry2.generationId,
|
|
229492
|
+
revision: retry2.revision,
|
|
229493
|
+
requestedAt: new Date().toISOString(),
|
|
229494
|
+
source: "api",
|
|
229495
|
+
refreshMode: retry2.refreshMode === "incremental" ? "incremental" : "full",
|
|
229496
|
+
...retry2.embeddingContextHash ? { embeddingContextHash: retry2.embeddingContextHash } : {},
|
|
229497
|
+
warningRetry: true
|
|
229498
|
+
};
|
|
229499
|
+
await queue2.add(stage, data, {
|
|
229500
|
+
...DEFAULT_JOB_OPTIONS,
|
|
229501
|
+
jobId: retryJobId,
|
|
229502
|
+
priority: SOURCE_PRIORITY.api
|
|
229503
|
+
});
|
|
229504
|
+
return {
|
|
229505
|
+
documentId,
|
|
229506
|
+
generationId: retry2.generationId,
|
|
229507
|
+
deduplicated: retry2.deduplicated,
|
|
229508
|
+
retriedStages: [
|
|
229509
|
+
...retry2.graph ? ["graph"] : [],
|
|
229510
|
+
...retry2.summarize ? ["summarize"] : []
|
|
229511
|
+
]
|
|
229512
|
+
};
|
|
229513
|
+
} catch (err) {
|
|
229514
|
+
const failedClaim = claimed;
|
|
229515
|
+
if (failedClaim) {
|
|
229516
|
+
await withTenant(ctx, (tx) => tx.update(documentPipelineRuns).set({
|
|
229517
|
+
status: "ready_with_warnings",
|
|
229518
|
+
finalizeStatus: "ready",
|
|
229519
|
+
...failedClaim.graph ? {
|
|
229520
|
+
graphStatus: "failed",
|
|
229521
|
+
graphErrorCode: "queue_enqueue_failed"
|
|
229522
|
+
} : {},
|
|
229523
|
+
...failedClaim.summarize ? {
|
|
229524
|
+
summarizeStatus: "failed",
|
|
229525
|
+
summarizeErrorCode: "queue_enqueue_failed"
|
|
229526
|
+
} : {},
|
|
229527
|
+
updatedAt: new Date
|
|
229528
|
+
}).where(and(eq(documentPipelineRuns.generationId, failedClaim.generationId), eq(documentPipelineRuns.status, "processing")))).catch(() => {
|
|
229529
|
+
return;
|
|
229530
|
+
});
|
|
229531
|
+
}
|
|
229532
|
+
logger3.error({ err, documentId }, "Failed to retry warning stages");
|
|
229533
|
+
set2.status = 500;
|
|
229534
|
+
return { error: "Failed to retry warning stages" };
|
|
229535
|
+
}
|
|
229221
229536
|
}).get("/documents/:id/knowledge-summary", async ({ params: params2, set: set2, request }) => {
|
|
229222
229537
|
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
229223
229538
|
if (!parsedParams.success) {
|
|
@@ -229307,6 +229622,8 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
229307
229622
|
embedStatus: documentPipelineRuns.embedStatus,
|
|
229308
229623
|
graphStatus: documentPipelineRuns.graphStatus,
|
|
229309
229624
|
summarizeStatus: documentPipelineRuns.summarizeStatus,
|
|
229625
|
+
graphErrorCode: documentPipelineRuns.graphErrorCode,
|
|
229626
|
+
summarizeErrorCode: documentPipelineRuns.summarizeErrorCode,
|
|
229310
229627
|
finalizeStatus: documentPipelineRuns.finalizeStatus,
|
|
229311
229628
|
totalBatches: documentPipelineRuns.totalBatches,
|
|
229312
229629
|
completedBatches: documentPipelineRuns.completedBatches,
|
|
@@ -229350,6 +229667,22 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
229350
229667
|
completed: run3.completedBatches,
|
|
229351
229668
|
failed: run3.failedBatches
|
|
229352
229669
|
},
|
|
229670
|
+
warnings: [
|
|
229671
|
+
...run3.graphStatus === "failed" ? [
|
|
229672
|
+
{
|
|
229673
|
+
stage: "graph",
|
|
229674
|
+
code: run3.graphErrorCode ?? "graph_failed",
|
|
229675
|
+
retryable: isRetryablePipelineError(run3.graphErrorCode ?? "graph_failed")
|
|
229676
|
+
}
|
|
229677
|
+
] : [],
|
|
229678
|
+
...run3.summarizeStatus === "failed" ? [
|
|
229679
|
+
{
|
|
229680
|
+
stage: "summarize",
|
|
229681
|
+
code: run3.summarizeErrorCode ?? "summary_failed",
|
|
229682
|
+
retryable: isRetryablePipelineError(run3.summarizeErrorCode ?? "summary_failed")
|
|
229683
|
+
}
|
|
229684
|
+
] : []
|
|
229685
|
+
],
|
|
229353
229686
|
updatedAt: run3.updatedAt
|
|
229354
229687
|
} : null
|
|
229355
229688
|
};
|
|
@@ -231076,8 +231409,9 @@ async function expandFromQueryPlan(plan, limit = 20) {
|
|
|
231076
231409
|
const boundedLimit = Math.max(1, Math.min(Math.floor(limit), 100));
|
|
231077
231410
|
try {
|
|
231078
231411
|
const cypher = buildQuerySeedCypher(terms, boundedLimit);
|
|
231079
|
-
const
|
|
231080
|
-
|
|
231412
|
+
const rows = await runCypher(sql6, cypher, {
|
|
231413
|
+
columns: "neighbor_id agtype, generation_id agtype, relation agtype, hops agtype"
|
|
231414
|
+
});
|
|
231081
231415
|
const seen = new Set;
|
|
231082
231416
|
const out = [];
|
|
231083
231417
|
for (const row of rows) {
|
|
@@ -231112,8 +231446,9 @@ async function expandResults(docIds, maxHops = DEFAULT_MAX_HOPS) {
|
|
|
231112
231446
|
const sql6 = await getGraphDbRequired();
|
|
231113
231447
|
try {
|
|
231114
231448
|
const cypher = buildTraversalCypher(seeds, clampedHops);
|
|
231115
|
-
const
|
|
231116
|
-
|
|
231449
|
+
const rows = await runCypher(sql6, cypher, {
|
|
231450
|
+
columns: "seed_id agtype, seed_generation_id agtype, neighbor_id agtype, generation_id agtype, relation agtype, hops agtype"
|
|
231451
|
+
});
|
|
231117
231452
|
for (const row of rows) {
|
|
231118
231453
|
const seedId = stripQuotes(String(row.seed_id ?? ""));
|
|
231119
231454
|
const neighborId = stripQuotes(String(row.neighbor_id ?? ""));
|
|
@@ -231185,6 +231520,13 @@ function buildCypherSql(cypher, columns2) {
|
|
|
231185
231520
|
const quoted = `$${tag2}$ ${cypher} $${tag2}$`;
|
|
231186
231521
|
return `SELECT * FROM cypher('docs_graph', ${quoted}) AS (${columns2})`;
|
|
231187
231522
|
}
|
|
231523
|
+
async function runCypher(sql6, cypher, options) {
|
|
231524
|
+
const queryString = buildCypherSql(cypher, options.columns);
|
|
231525
|
+
return sql6.begin(async (tx) => {
|
|
231526
|
+
await tx.unsafe("SET LOCAL search_path = ag_catalog, public");
|
|
231527
|
+
return await tx.unsafe(queryString);
|
|
231528
|
+
});
|
|
231529
|
+
}
|
|
231188
231530
|
function stripQuotes(value) {
|
|
231189
231531
|
const trimmed = value.trim();
|
|
231190
231532
|
if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
@@ -231571,7 +231913,7 @@ function createHealthRoutes(dependencies) {
|
|
|
231571
231913
|
set2.status = 503;
|
|
231572
231914
|
return {
|
|
231573
231915
|
status: readiness.status === "unhealthy" ? "unhealthy" : "ok",
|
|
231574
|
-
service: "
|
|
231916
|
+
service: "docsmint",
|
|
231575
231917
|
timestamp: new Date().toISOString(),
|
|
231576
231918
|
database: databaseAvailable ? "ok" : "error",
|
|
231577
231919
|
redis: redisAvailable ? "ok" : "error",
|
|
@@ -231876,9 +232218,10 @@ async function retrieveGraphCandidates(ctx, request, adapters = {}) {
|
|
|
231876
232218
|
const requestedSeeds = dedupe2(request.documentSeeds).slice(0, config3.SEARCH_GRAPH_SEED_LIMIT);
|
|
231877
232219
|
const visibilityScope = request.visibilityScope ?? _buildGraphVisibilityScope(ctx);
|
|
231878
232220
|
const tenantTransaction = adapters.withTenant ?? withTenant;
|
|
231879
|
-
const
|
|
232221
|
+
const categoryId = visibilityScope.kind === "share" ? undefined : request.categoryId;
|
|
232222
|
+
const authorizedSeeds = requestedSeeds.length === 0 ? new Set : await resolveVisibleIds(ctx, requestedSeeds, adapters.visibleDocumentIds, visibilityScope, categoryId, tenantTransaction);
|
|
231880
232223
|
const seeds = requestedSeeds.filter((id3) => authorizedSeeds.has(id3));
|
|
231881
|
-
const activeSeedGenerations = await resolveActiveGenerations(ctx, seeds, adapters.visibleDocumentGenerations, visibilityScope,
|
|
232224
|
+
const activeSeedGenerations = await resolveActiveGenerations(ctx, seeds, adapters.visibleDocumentGenerations, visibilityScope, categoryId, Boolean(adapters.visibleDocumentIds), tenantTransaction);
|
|
231882
232225
|
const expand2 = adapters.expandResults ?? expandResults;
|
|
231883
232226
|
const expandQuery = adapters.expandFromQueryPlan ?? expandFromQueryPlan;
|
|
231884
232227
|
let related = [];
|
|
@@ -231902,8 +232245,8 @@ async function retrieveGraphCandidates(ctx, request, adapters = {}) {
|
|
|
231902
232245
|
});
|
|
231903
232246
|
});
|
|
231904
232247
|
const allIds = dedupe2(relatedWithCurrentSeed.map((candidate) => candidate.docId));
|
|
231905
|
-
const visible = await resolveVisibleIds(ctx, allIds, adapters.visibleDocumentIds, visibilityScope,
|
|
231906
|
-
const activeGenerations = await resolveActiveGenerations(ctx, allIds, adapters.visibleDocumentGenerations, visibilityScope,
|
|
232248
|
+
const visible = await resolveVisibleIds(ctx, allIds, adapters.visibleDocumentIds, visibilityScope, categoryId, tenantTransaction);
|
|
232249
|
+
const activeGenerations = await resolveActiveGenerations(ctx, allIds, adapters.visibleDocumentGenerations, visibilityScope, categoryId, Boolean(adapters.visibleDocumentIds), tenantTransaction);
|
|
231907
232250
|
const unique = new Map;
|
|
231908
232251
|
for (const candidate of relatedWithCurrentSeed) {
|
|
231909
232252
|
if (!visible.has(candidate.docId))
|
|
@@ -232168,19 +232511,59 @@ function resolveChatProviderKey(baseUrl, explicitKey, sharedOpenRouterKey) {
|
|
|
232168
232511
|
return "";
|
|
232169
232512
|
return sharedOpenRouterKey?.trim() ?? "";
|
|
232170
232513
|
}
|
|
232514
|
+
function uniqueChatProviders(providers) {
|
|
232515
|
+
const seen = new Set;
|
|
232516
|
+
const unique = [];
|
|
232517
|
+
for (const provider of providers) {
|
|
232518
|
+
if (!provider?.baseUrl || !provider.model)
|
|
232519
|
+
continue;
|
|
232520
|
+
const identity = [
|
|
232521
|
+
provider.baseUrl,
|
|
232522
|
+
provider.model,
|
|
232523
|
+
provider.apiKey ?? ""
|
|
232524
|
+
].join("|");
|
|
232525
|
+
if (seen.has(identity))
|
|
232526
|
+
continue;
|
|
232527
|
+
seen.add(identity);
|
|
232528
|
+
unique.push(provider);
|
|
232529
|
+
}
|
|
232530
|
+
return unique;
|
|
232531
|
+
}
|
|
232171
232532
|
async function requestStructuredChat(options) {
|
|
232172
|
-
const
|
|
232533
|
+
const attempt = await requestStructuredChatDetailed(options);
|
|
232534
|
+
return attempt.ok ? attempt.result : null;
|
|
232535
|
+
}
|
|
232536
|
+
async function requestStructuredChatDetailed(options) {
|
|
232537
|
+
const providers = uniqueChatProviders([
|
|
232538
|
+
options.primary,
|
|
232539
|
+
options.fallback,
|
|
232540
|
+
...options.fallbacks ?? []
|
|
232541
|
+
]);
|
|
232542
|
+
let lastError = new PipelineProviderError("provider_unavailable", false, "No chat provider was configured");
|
|
232173
232543
|
for (const provider of providers) {
|
|
232174
232544
|
try {
|
|
232175
232545
|
const raw = await requestChatContent(provider, options);
|
|
232176
|
-
|
|
232546
|
+
let parsed;
|
|
232547
|
+
try {
|
|
232548
|
+
parsed = parseJson(raw);
|
|
232549
|
+
} catch {
|
|
232550
|
+
lastError = new PipelineProviderError("invalid_provider_response", true);
|
|
232551
|
+
continue;
|
|
232552
|
+
}
|
|
232177
232553
|
const result = options.outputSchema.safeParse(parsed);
|
|
232178
|
-
if (!result.success)
|
|
232554
|
+
if (!result.success) {
|
|
232555
|
+
lastError = new PipelineProviderError("permanent_validation_failure", false);
|
|
232179
232556
|
continue;
|
|
232180
|
-
|
|
232181
|
-
|
|
232557
|
+
}
|
|
232558
|
+
return {
|
|
232559
|
+
ok: true,
|
|
232560
|
+
result: { data: result.data, model: provider.model }
|
|
232561
|
+
};
|
|
232562
|
+
} catch (error53) {
|
|
232563
|
+
lastError = error53 instanceof PipelineProviderError ? error53 : error53 instanceof Error && error53.name === "AbortError" ? new PipelineProviderError("provider_timeout", true) : new PipelineProviderError("provider_failure", true);
|
|
232564
|
+
}
|
|
232182
232565
|
}
|
|
232183
|
-
return
|
|
232566
|
+
return { ok: false, error: lastError };
|
|
232184
232567
|
}
|
|
232185
232568
|
async function requestChatContent(provider, options) {
|
|
232186
232569
|
const controller = new AbortController;
|
|
@@ -232205,11 +232588,16 @@ async function requestChatContent(provider, options) {
|
|
|
232205
232588
|
})
|
|
232206
232589
|
});
|
|
232207
232590
|
if (!response.ok)
|
|
232208
|
-
throw new
|
|
232209
|
-
|
|
232591
|
+
throw new PipelineProviderError("provider_failure", true, `chat provider returned ${response.status}`);
|
|
232592
|
+
let body;
|
|
232593
|
+
try {
|
|
232594
|
+
body = await response.json();
|
|
232595
|
+
} catch {
|
|
232596
|
+
throw new PipelineProviderError("invalid_provider_response", true);
|
|
232597
|
+
}
|
|
232210
232598
|
const content50 = body.choices?.[0]?.message?.content;
|
|
232211
232599
|
if (typeof content50 !== "string" || !content50.trim()) {
|
|
232212
|
-
throw new
|
|
232600
|
+
throw new PipelineProviderError("invalid_provider_response", true);
|
|
232213
232601
|
}
|
|
232214
232602
|
return content50;
|
|
232215
232603
|
} finally {
|
|
@@ -232241,13 +232629,18 @@ async function expandQuery(plan, scope) {
|
|
|
232241
232629
|
const tenantScope = typeof scope === "string" ? scope.trim() : (scope.tenantScope ?? scope.tenantId ?? scope.ownerId ?? "").trim();
|
|
232242
232630
|
if (!tenantScope || !plan.normalized.trim())
|
|
232243
232631
|
return null;
|
|
232244
|
-
const
|
|
232245
|
-
|
|
232246
|
-
|
|
232247
|
-
|
|
232248
|
-
|
|
232249
|
-
|
|
232250
|
-
|
|
232632
|
+
const chain3 = uniqueChatProviders([
|
|
232633
|
+
providerConfig(config3.SEARCH_EXPANSION_BASE_URL, config3.SEARCH_EXPANSION_API_KEY, config3.SEARCH_EXPANSION_MODEL),
|
|
232634
|
+
providerConfig(config3.SEARCH_EXPANSION_FALLBACK_BASE_URL, config3.SEARCH_EXPANSION_FALLBACK_API_KEY, config3.SEARCH_EXPANSION_FALLBACK_MODEL),
|
|
232635
|
+
providerConfig(config3.SEARCH_EXPANSION_FALLBACK_2_BASE_URL, config3.SEARCH_EXPANSION_FALLBACK_2_API_KEY, config3.SEARCH_EXPANSION_FALLBACK_2_MODEL)
|
|
232636
|
+
]);
|
|
232637
|
+
const providerTimeoutMs = Math.max(250, Math.floor(config3.SEARCH_EXPANSION_TIMEOUT_MS / Math.max(1, chain3.length)));
|
|
232638
|
+
for (const provider of chain3)
|
|
232639
|
+
provider.timeoutMs = providerTimeoutMs;
|
|
232640
|
+
const [primary, ...fallbacks] = chain3;
|
|
232641
|
+
if (!primary)
|
|
232642
|
+
return null;
|
|
232643
|
+
const key = await expansionCacheKey(tenantScope, plan.normalized, primary, fallbacks);
|
|
232251
232644
|
const ttl = config3.SEARCH_EXPANSION_CACHE_TTL_SECONDS;
|
|
232252
232645
|
if (ttl > 0) {
|
|
232253
232646
|
try {
|
|
@@ -232272,7 +232665,7 @@ async function expandQuery(plan, scope) {
|
|
|
232272
232665
|
}
|
|
232273
232666
|
const result = await requestStructuredChat({
|
|
232274
232667
|
primary,
|
|
232275
|
-
|
|
232668
|
+
fallbacks,
|
|
232276
232669
|
messages: [
|
|
232277
232670
|
{
|
|
232278
232671
|
role: "system",
|
|
@@ -232352,12 +232745,15 @@ var cachedResultSchema = exports_external2.object({
|
|
|
232352
232745
|
}),
|
|
232353
232746
|
model: exports_external2.string()
|
|
232354
232747
|
});
|
|
232355
|
-
async function expansionCacheKey(tenantScope, normalizedQuery, primary =
|
|
232748
|
+
async function expansionCacheKey(tenantScope, normalizedQuery, primary = {
|
|
232749
|
+
baseUrl: config3.SEARCH_EXPANSION_BASE_URL,
|
|
232750
|
+
model: config3.SEARCH_EXPANSION_MODEL,
|
|
232751
|
+
timeoutMs: config3.SEARCH_EXPANSION_TIMEOUT_MS
|
|
232752
|
+
}, fallbacks = []) {
|
|
232356
232753
|
const modelProfile = [
|
|
232357
232754
|
primary.baseUrl,
|
|
232358
232755
|
primary.model,
|
|
232359
|
-
|
|
232360
|
-
fallback?.model ?? ""
|
|
232756
|
+
...fallbacks.flatMap((provider) => [provider.baseUrl, provider.model])
|
|
232361
232757
|
].join("|");
|
|
232362
232758
|
const input = [
|
|
232363
232759
|
tenantScope,
|
|
@@ -232369,6 +232765,8 @@ async function expansionCacheKey(tenantScope, normalizedQuery, primary = provide
|
|
|
232369
232765
|
return `hiai-docs:search:expansion:${digest4}`;
|
|
232370
232766
|
}
|
|
232371
232767
|
function providerConfig(baseUrl, explicitKey, model) {
|
|
232768
|
+
if (!baseUrl || !model)
|
|
232769
|
+
return;
|
|
232372
232770
|
return {
|
|
232373
232771
|
baseUrl,
|
|
232374
232772
|
model,
|
|
@@ -232396,6 +232794,179 @@ function normalizeVariant(value) {
|
|
|
232396
232794
|
return value.normalize("NFKC").trim().replace(/\s+/g, " ").toLocaleLowerCase();
|
|
232397
232795
|
}
|
|
232398
232796
|
|
|
232797
|
+
// ../../backend/src/search/rerank.ts
|
|
232798
|
+
var DEFAULT_RERANK_MAX_CHARS = 1500;
|
|
232799
|
+
function normalizeRerankText(title, content50, maxChars = DEFAULT_RERANK_MAX_CHARS) {
|
|
232800
|
+
const cleaned = `${title.trim()}
|
|
232801
|
+
${content50}`.replace(/data:[^\s)>]+/gi, " ").replace(/\s+/g, " ").trim();
|
|
232802
|
+
if (cleaned.length <= maxChars)
|
|
232803
|
+
return cleaned;
|
|
232804
|
+
return cleaned.slice(0, maxChars);
|
|
232805
|
+
}
|
|
232806
|
+
function applyRerankOrder(ranked, hits, topN) {
|
|
232807
|
+
if (ranked.length === 0 || hits.length === 0 || topN < 1)
|
|
232808
|
+
return ranked;
|
|
232809
|
+
const window2 = ranked.slice(0, Math.min(topN, ranked.length));
|
|
232810
|
+
const rest = ranked.slice(window2.length);
|
|
232811
|
+
const byId = new Map(window2.map((item) => [item.documentId, item]));
|
|
232812
|
+
const seen = new Set;
|
|
232813
|
+
const reordered = [];
|
|
232814
|
+
const sortedHits = [...hits].sort((left, right) => {
|
|
232815
|
+
if (right.score !== left.score)
|
|
232816
|
+
return right.score - left.score;
|
|
232817
|
+
return left.rank - right.rank;
|
|
232818
|
+
});
|
|
232819
|
+
for (const hit of sortedHits) {
|
|
232820
|
+
const item = byId.get(hit.id);
|
|
232821
|
+
if (!item || seen.has(hit.id))
|
|
232822
|
+
continue;
|
|
232823
|
+
seen.add(hit.id);
|
|
232824
|
+
reordered.push(item);
|
|
232825
|
+
}
|
|
232826
|
+
for (const item of window2) {
|
|
232827
|
+
if (seen.has(item.documentId))
|
|
232828
|
+
continue;
|
|
232829
|
+
reordered.push(item);
|
|
232830
|
+
}
|
|
232831
|
+
return [...reordered, ...rest];
|
|
232832
|
+
}
|
|
232833
|
+
|
|
232834
|
+
// ../../backend/src/search/rerank-provider.ts
|
|
232835
|
+
init_config();
|
|
232836
|
+
init_logger3();
|
|
232837
|
+
function configuredSlots() {
|
|
232838
|
+
const slots = [
|
|
232839
|
+
{
|
|
232840
|
+
baseUrl: config3.SEARCH_RERANK_BASE_URL,
|
|
232841
|
+
apiKey: config3.SEARCH_RERANK_API_KEY,
|
|
232842
|
+
model: config3.SEARCH_RERANK_MODEL
|
|
232843
|
+
},
|
|
232844
|
+
{
|
|
232845
|
+
baseUrl: config3.SEARCH_RERANK_FALLBACK_BASE_URL,
|
|
232846
|
+
apiKey: config3.SEARCH_RERANK_FALLBACK_API_KEY,
|
|
232847
|
+
model: config3.SEARCH_RERANK_FALLBACK_MODEL
|
|
232848
|
+
},
|
|
232849
|
+
{
|
|
232850
|
+
baseUrl: config3.SEARCH_RERANK_FALLBACK_2_BASE_URL,
|
|
232851
|
+
apiKey: config3.SEARCH_RERANK_FALLBACK_2_API_KEY,
|
|
232852
|
+
model: config3.SEARCH_RERANK_FALLBACK_2_MODEL
|
|
232853
|
+
}
|
|
232854
|
+
];
|
|
232855
|
+
const seen = new Set;
|
|
232856
|
+
const unique = [];
|
|
232857
|
+
for (const slot of slots) {
|
|
232858
|
+
if (!slot.baseUrl || !slot.model)
|
|
232859
|
+
continue;
|
|
232860
|
+
const identity = `${slot.baseUrl}|${slot.model}`;
|
|
232861
|
+
if (seen.has(identity))
|
|
232862
|
+
continue;
|
|
232863
|
+
seen.add(identity);
|
|
232864
|
+
unique.push({
|
|
232865
|
+
baseUrl: slot.baseUrl,
|
|
232866
|
+
model: slot.model,
|
|
232867
|
+
apiKey: resolveChatProviderKey(slot.baseUrl, slot.apiKey, config3.OPENROUTER_API_KEY)
|
|
232868
|
+
});
|
|
232869
|
+
}
|
|
232870
|
+
return unique;
|
|
232871
|
+
}
|
|
232872
|
+
function rerankUrl(baseUrl) {
|
|
232873
|
+
const normalized = baseUrl.replace(/\/$/, "");
|
|
232874
|
+
return normalized.endsWith("/rerank") ? normalized : `${normalized}/rerank`;
|
|
232875
|
+
}
|
|
232876
|
+
async function requestRerank(input) {
|
|
232877
|
+
const candidates = input.candidates.map((candidate, originalIndex) => ({
|
|
232878
|
+
id: candidate.id,
|
|
232879
|
+
document: candidate.text.trim(),
|
|
232880
|
+
originalIndex
|
|
232881
|
+
})).filter((candidate) => candidate.document.length > 0);
|
|
232882
|
+
if (candidates.length === 0)
|
|
232883
|
+
return null;
|
|
232884
|
+
const topN = Math.min(input.topN ?? config3.SEARCH_RERANK_TOP_N, candidates.length);
|
|
232885
|
+
const timeoutMs = config3.SEARCH_RERANK_TIMEOUT_MS;
|
|
232886
|
+
const started = performance.now();
|
|
232887
|
+
incrementCounterBy(METRIC_NAMES.SEARCH_RERANK_CANDIDATES_TOTAL, candidates.length);
|
|
232888
|
+
try {
|
|
232889
|
+
for (const slot of configuredSlots()) {
|
|
232890
|
+
try {
|
|
232891
|
+
const hits = await callRerankSlot(slot, input.query, candidates, {
|
|
232892
|
+
topN,
|
|
232893
|
+
timeoutMs
|
|
232894
|
+
});
|
|
232895
|
+
if (hits.length === 0)
|
|
232896
|
+
continue;
|
|
232897
|
+
incrementCounter(METRIC_NAMES.SEARCH_RERANK_SUCCESS_TOTAL);
|
|
232898
|
+
return { hits, model: slot.model };
|
|
232899
|
+
} catch (error53) {
|
|
232900
|
+
logger3.warn({ err: error53, model: slot.model }, "Rerank provider failed, trying next configured provider");
|
|
232901
|
+
}
|
|
232902
|
+
}
|
|
232903
|
+
incrementCounter(METRIC_NAMES.SEARCH_RERANK_FALLBACK_TOTAL);
|
|
232904
|
+
return null;
|
|
232905
|
+
} finally {
|
|
232906
|
+
recordDuration(METRIC_NAMES.SEARCH_RERANK_DURATION_MS, performance.now() - started);
|
|
232907
|
+
}
|
|
232908
|
+
}
|
|
232909
|
+
async function callRerankSlot(slot, query, candidates, options) {
|
|
232910
|
+
const controller = new AbortController;
|
|
232911
|
+
const timer2 = setTimeout(() => controller.abort(), options.timeoutMs);
|
|
232912
|
+
const headers = {
|
|
232913
|
+
"Content-Type": "application/json"
|
|
232914
|
+
};
|
|
232915
|
+
if (slot.apiKey)
|
|
232916
|
+
headers.Authorization = `Bearer ${slot.apiKey}`;
|
|
232917
|
+
try {
|
|
232918
|
+
const response = await fetch(rerankUrl(slot.baseUrl), {
|
|
232919
|
+
method: "POST",
|
|
232920
|
+
headers,
|
|
232921
|
+
signal: controller.signal,
|
|
232922
|
+
body: JSON.stringify({
|
|
232923
|
+
model: slot.model,
|
|
232924
|
+
query,
|
|
232925
|
+
documents: candidates.map((candidate) => candidate.document),
|
|
232926
|
+
top_n: options.topN
|
|
232927
|
+
})
|
|
232928
|
+
});
|
|
232929
|
+
if (!response.ok) {
|
|
232930
|
+
throw new Error(`rerank provider returned ${response.status}`);
|
|
232931
|
+
}
|
|
232932
|
+
const body = await response.json();
|
|
232933
|
+
const hits = [];
|
|
232934
|
+
const seenIds = new Set;
|
|
232935
|
+
for (const [rank, row] of (body.results ?? []).entries()) {
|
|
232936
|
+
if (typeof row.index !== "number" || !Number.isInteger(row.index) || row.index < 0)
|
|
232937
|
+
continue;
|
|
232938
|
+
const candidate = candidates[row.index];
|
|
232939
|
+
if (!candidate || seenIds.has(candidate.id))
|
|
232940
|
+
continue;
|
|
232941
|
+
seenIds.add(candidate.id);
|
|
232942
|
+
const score = typeof row.relevance_score === "number" && Number.isFinite(row.relevance_score) ? row.relevance_score : 0;
|
|
232943
|
+
hits.push({ id: candidate.id, score, rank: rank + 1 });
|
|
232944
|
+
}
|
|
232945
|
+
return hits;
|
|
232946
|
+
} finally {
|
|
232947
|
+
clearTimeout(timer2);
|
|
232948
|
+
}
|
|
232949
|
+
}
|
|
232950
|
+
|
|
232951
|
+
// ../../backend/src/search/rerank-texts.ts
|
|
232952
|
+
init_schema2();
|
|
232953
|
+
init_drizzle_orm();
|
|
232954
|
+
init_config();
|
|
232955
|
+
async function loadRerankTexts(ctx, documentIds, maxChars = config3.SEARCH_RERANK_MAX_CHARS) {
|
|
232956
|
+
const texts = new Map;
|
|
232957
|
+
if (documentIds.length === 0)
|
|
232958
|
+
return texts;
|
|
232959
|
+
const rows = await withTenant(ctx, async (tx) => tx.select({
|
|
232960
|
+
id: documents.id,
|
|
232961
|
+
title: documents.title,
|
|
232962
|
+
content: documents.content
|
|
232963
|
+
}).from(documents).where(and(isNull(documents.deletedAt), inArray(documents.id, documentIds))));
|
|
232964
|
+
for (const row of rows) {
|
|
232965
|
+
texts.set(row.id, normalizeRerankText(row.title, row.content ?? "", maxChars));
|
|
232966
|
+
}
|
|
232967
|
+
return texts;
|
|
232968
|
+
}
|
|
232969
|
+
|
|
232399
232970
|
// ../../backend/src/search/retrievers.ts
|
|
232400
232971
|
init_drizzle_orm();
|
|
232401
232972
|
init_config();
|
|
@@ -232714,6 +233285,8 @@ async function searchDocuments(ctx, request, adapters = {}) {
|
|
|
232714
233285
|
const embeddingProvider = adapters.getEmbedding ?? getEmbedding;
|
|
232715
233286
|
const expand2 = adapters.expand ?? expandQuery;
|
|
232716
233287
|
const retrieveGraph = adapters.retrieveGraph ?? retrieveGraphCandidates;
|
|
233288
|
+
const rerankDocuments = adapters.rerank ?? requestRerank;
|
|
233289
|
+
const loadTexts = adapters.loadRerankTexts ?? loadRerankTexts;
|
|
232717
233290
|
const filterRanked = adapters.filterRanked ?? applySearchFilters;
|
|
232718
233291
|
const channelErrors = {};
|
|
232719
233292
|
const embeddingCache = new Map;
|
|
@@ -232807,27 +233380,36 @@ async function searchDocuments(ctx, request, adapters = {}) {
|
|
|
232807
233380
|
model: expansionModel,
|
|
232808
233381
|
primaryModel: config3.SEARCH_EXPANSION_MODEL,
|
|
232809
233382
|
fallbackModel: config3.SEARCH_EXPANSION_FALLBACK_MODEL,
|
|
233383
|
+
fallback2Model: config3.SEARCH_EXPANSION_FALLBACK_2_MODEL,
|
|
232810
233384
|
estimatedCostMicrounits: expansionUsed ? config3.SEARCH_EXPANSION_ESTIMATED_COST_MICROUNITS : 0
|
|
232811
233385
|
});
|
|
232812
233386
|
recordDuration(METRIC_NAMES.SEARCH_EXPANDED_DURATION_MS, performance.now() - expandedStarted);
|
|
232813
233387
|
}
|
|
232814
|
-
const
|
|
233388
|
+
const rrfOptions = {
|
|
232815
233389
|
rrfK: config3.SEARCH_RRF_K,
|
|
232816
233390
|
exactBoost: config3.SEARCH_EXACT_BOOST,
|
|
232817
233391
|
channelAgreementBoost: config3.SEARCH_CHANNEL_AGREEMENT_BOOST,
|
|
232818
233392
|
graphMaxContribution: config3.SEARCH_GRAPH_MAX_CONTRIBUTION,
|
|
232819
233393
|
vectorMinSimilarity: config3.SEARCH_VECTOR_MIN_SIMILARITY
|
|
232820
|
-
}
|
|
233394
|
+
};
|
|
233395
|
+
const direct = fuseCandidates([...fast, ...expanded].flatMap((result) => result.candidates), rrfOptions);
|
|
232821
233396
|
const graphPlan = expandedPlan ?? plan;
|
|
232822
|
-
const
|
|
233397
|
+
const graphAttempted = request.graphEnabled !== false;
|
|
233398
|
+
const rerankEnabled = request.rerankEnabled ?? config3.SEARCH_RERANK_ENABLED;
|
|
233399
|
+
const graphPosition = request.rerankGraphPosition ?? config3.SEARCH_RERANK_GRAPH_POSITION;
|
|
232823
233400
|
let graph = [];
|
|
232824
233401
|
let graphFailed = false;
|
|
232825
|
-
|
|
232826
|
-
|
|
233402
|
+
let ranked;
|
|
233403
|
+
let rerankAttempted = false;
|
|
233404
|
+
let rerankUsed = false;
|
|
233405
|
+
let rerankFallback = false;
|
|
233406
|
+
let rerankModel;
|
|
233407
|
+
const runGraph = async (seeds) => {
|
|
232827
233408
|
const graphStarted = performance.now();
|
|
233409
|
+
let next = [];
|
|
232828
233410
|
try {
|
|
232829
|
-
|
|
232830
|
-
documentSeeds:
|
|
233411
|
+
next = await retrieveGraph(ctx, {
|
|
233412
|
+
documentSeeds: seeds,
|
|
232831
233413
|
queryPlan: graphPlan,
|
|
232832
233414
|
limit: Math.min(limit * 2, config3.SEARCH_GRAPH_RESULT_LIMIT),
|
|
232833
233415
|
maxHops: request.maxGraphHops ?? config3.SEARCH_GRAPH_MAX_HOPS,
|
|
@@ -232837,25 +233419,82 @@ async function searchDocuments(ctx, request, adapters = {}) {
|
|
|
232837
233419
|
} catch {
|
|
232838
233420
|
graphFailed = true;
|
|
232839
233421
|
}
|
|
232840
|
-
|
|
233422
|
+
next = restrictCandidates(next, request.documentIds);
|
|
232841
233423
|
recordSearchChannelMetrics({
|
|
232842
233424
|
channel: "graph",
|
|
232843
233425
|
durationMs: performance.now() - graphStarted,
|
|
232844
|
-
candidateCount:
|
|
233426
|
+
candidateCount: next.length,
|
|
232845
233427
|
errorCode: graphFailed ? "query_failed" : undefined
|
|
232846
233428
|
});
|
|
233429
|
+
return next;
|
|
233430
|
+
};
|
|
233431
|
+
const seedIds = (items2) => items2.slice(0, config3.SEARCH_GRAPH_SEED_LIMIT).map((item) => item.documentId);
|
|
233432
|
+
const applyOptionalRerank = async (items2) => {
|
|
233433
|
+
if (!rerankEnabled || items2.length === 0)
|
|
233434
|
+
return items2;
|
|
233435
|
+
rerankAttempted = true;
|
|
233436
|
+
try {
|
|
233437
|
+
const topN = Math.min(config3.SEARCH_RERANK_TOP_N, items2.length);
|
|
233438
|
+
const window2 = items2.slice(0, topN);
|
|
233439
|
+
const texts = await loadTexts(ctx, window2.map((item) => item.documentId));
|
|
233440
|
+
const candidates = window2.map((item, index2) => ({
|
|
233441
|
+
id: item.documentId,
|
|
233442
|
+
text: texts.get(item.documentId) ?? "",
|
|
233443
|
+
sourceRank: index2 + 1,
|
|
233444
|
+
rrfScore: item.score
|
|
233445
|
+
}));
|
|
233446
|
+
if (candidates.every((candidate) => candidate.text.length === 0)) {
|
|
233447
|
+
rerankFallback = true;
|
|
233448
|
+
return items2;
|
|
233449
|
+
}
|
|
233450
|
+
const result = await rerankDocuments({
|
|
233451
|
+
query: plan.original,
|
|
233452
|
+
candidates,
|
|
233453
|
+
topN
|
|
233454
|
+
});
|
|
233455
|
+
if (!result || result.hits.length === 0) {
|
|
233456
|
+
rerankFallback = true;
|
|
233457
|
+
return items2;
|
|
233458
|
+
}
|
|
233459
|
+
rerankUsed = true;
|
|
233460
|
+
rerankModel = result.model;
|
|
233461
|
+
return applyRerankOrder(items2, result.hits, topN);
|
|
233462
|
+
} catch {
|
|
233463
|
+
rerankFallback = true;
|
|
233464
|
+
return items2;
|
|
233465
|
+
}
|
|
233466
|
+
};
|
|
233467
|
+
if (!rerankEnabled) {
|
|
233468
|
+
if (graphAttempted)
|
|
233469
|
+
graph = await runGraph(seedIds(direct));
|
|
233470
|
+
ranked = fuseCandidates([
|
|
233471
|
+
...fast.flatMap((result) => result.candidates),
|
|
233472
|
+
...expanded.flatMap((result) => result.candidates),
|
|
233473
|
+
...graph
|
|
233474
|
+
], rrfOptions);
|
|
233475
|
+
} else if (graphPosition === "before") {
|
|
233476
|
+
if (graphAttempted)
|
|
233477
|
+
graph = await runGraph(seedIds(direct));
|
|
233478
|
+
ranked = fuseCandidates([
|
|
233479
|
+
...fast.flatMap((result) => result.candidates),
|
|
233480
|
+
...expanded.flatMap((result) => result.candidates),
|
|
233481
|
+
...graph
|
|
233482
|
+
], rrfOptions);
|
|
233483
|
+
ranked = await applyOptionalRerank(ranked);
|
|
233484
|
+
} else {
|
|
233485
|
+
ranked = await applyOptionalRerank(direct);
|
|
233486
|
+
if (graphAttempted)
|
|
233487
|
+
graph = await runGraph(seedIds(ranked));
|
|
233488
|
+
ranked = annotateGraphChannels(ranked, graph);
|
|
233489
|
+
const present = new Set(ranked.map((item) => item.documentId));
|
|
233490
|
+
const extra = graph.filter((candidate) => !present.has(candidate.documentId));
|
|
233491
|
+
if (extra.length > 0) {
|
|
233492
|
+
ranked = [
|
|
233493
|
+
...ranked,
|
|
233494
|
+
...fuseCandidates(extra, rrfOptions).filter((item) => !present.has(item.documentId))
|
|
233495
|
+
];
|
|
233496
|
+
}
|
|
232847
233497
|
}
|
|
232848
|
-
const ranked = fuseCandidates([
|
|
232849
|
-
...fast.flatMap((result) => result.candidates),
|
|
232850
|
-
...expanded.flatMap((result) => result.candidates),
|
|
232851
|
-
...graph
|
|
232852
|
-
], {
|
|
232853
|
-
rrfK: config3.SEARCH_RRF_K,
|
|
232854
|
-
exactBoost: config3.SEARCH_EXACT_BOOST,
|
|
232855
|
-
channelAgreementBoost: config3.SEARCH_CHANNEL_AGREEMENT_BOOST,
|
|
232856
|
-
graphMaxContribution: config3.SEARCH_GRAPH_MAX_CONTRIBUTION,
|
|
232857
|
-
vectorMinSimilarity: config3.SEARCH_VECTOR_MIN_SIMILARITY
|
|
232858
|
-
});
|
|
232859
233498
|
const filters = request.filters;
|
|
232860
233499
|
const hasFilters = Boolean(filters && Object.values(filters).some((value) => Array.isArray(value) ? value.length > 0 : value !== undefined));
|
|
232861
233500
|
const filtered = hasFilters || request.visibilityScope !== undefined ? await filterRanked(ctx, ranked, filters ?? {}, request.visibilityScope) : ranked;
|
|
@@ -232881,7 +233520,11 @@ async function searchDocuments(ctx, request, adapters = {}) {
|
|
|
232881
233520
|
graphAttempted,
|
|
232882
233521
|
graphFailed,
|
|
232883
233522
|
graphContribution,
|
|
232884
|
-
confidenceReasons: confidence.reasons
|
|
233523
|
+
confidenceReasons: confidence.reasons,
|
|
233524
|
+
rerankAttempted,
|
|
233525
|
+
rerankUsed,
|
|
233526
|
+
rerankFallback,
|
|
233527
|
+
...rerankModel ? { rerankModel } : {}
|
|
232885
233528
|
};
|
|
232886
233529
|
return {
|
|
232887
233530
|
items,
|
|
@@ -233049,6 +233692,24 @@ function clampPage(value) {
|
|
|
233049
233692
|
return 1;
|
|
233050
233693
|
return Math.max(1, Math.floor(value));
|
|
233051
233694
|
}
|
|
233695
|
+
function annotateGraphChannels(items, graphCandidates) {
|
|
233696
|
+
if (graphCandidates.length === 0)
|
|
233697
|
+
return items;
|
|
233698
|
+
const graphIds = new Set(graphCandidates.map((candidate) => candidate.documentId));
|
|
233699
|
+
return items.map((item) => {
|
|
233700
|
+
if (!graphIds.has(item.documentId) || item.channels.includes("graph")) {
|
|
233701
|
+
return item;
|
|
233702
|
+
}
|
|
233703
|
+
return {
|
|
233704
|
+
...item,
|
|
233705
|
+
channels: [...item.channels, "graph"],
|
|
233706
|
+
explanations: [
|
|
233707
|
+
...item.explanations,
|
|
233708
|
+
{ channel: "graph", label: "Related document" }
|
|
233709
|
+
]
|
|
233710
|
+
};
|
|
233711
|
+
});
|
|
233712
|
+
}
|
|
233052
233713
|
|
|
233053
233714
|
// ../../backend/src/api/routes/search.ts
|
|
233054
233715
|
var BENCHMARK_SEARCH_PROFILE_HEADER = "x-docsmint-search-profile";
|
|
@@ -235540,7 +236201,11 @@ async function extractEntities(chunkText2, documentId, options = {}) {
|
|
|
235540
236201
|
} catch (err) {
|
|
235541
236202
|
await releaseDedupClaim();
|
|
235542
236203
|
logger3.warn({ err, documentId }, "Entity extraction LLM call failed \u2014 skipping");
|
|
235543
|
-
return {
|
|
236204
|
+
return {
|
|
236205
|
+
status: "failed",
|
|
236206
|
+
warning: err instanceof Error && err.message ? err.message : "provider_failure",
|
|
236207
|
+
entities: []
|
|
236208
|
+
};
|
|
235544
236209
|
}
|
|
235545
236210
|
}
|
|
235546
236211
|
if (entities === null) {
|
|
@@ -235564,7 +236229,42 @@ async function extractEntities(chunkText2, documentId, options = {}) {
|
|
|
235564
236229
|
return { status: "ready", entities };
|
|
235565
236230
|
}
|
|
235566
236231
|
function hasConfiguredGraphProvider(options) {
|
|
235567
|
-
return Boolean(options.llmBaseUrl || config3.GRAPH_EXTRACT_BASE_URL || config3.GRAPH_EXTRACT_FALLBACK_BASE_URL);
|
|
236232
|
+
return Boolean(options.llmBaseUrl || config3.GRAPH_EXTRACT_BASE_URL || config3.GRAPH_EXTRACT_FALLBACK_BASE_URL || config3.GRAPH_EXTRACT_FALLBACK_2_BASE_URL);
|
|
236233
|
+
}
|
|
236234
|
+
function graphExtractProviders(options) {
|
|
236235
|
+
const timeoutMs = config3.GRAPH_EXTRACT_TIMEOUT_MS;
|
|
236236
|
+
const primaryModel = options.llmModel ?? config3.GRAPH_EXTRACT_MODEL ?? "gpt-4o-mini";
|
|
236237
|
+
const slots = [
|
|
236238
|
+
{
|
|
236239
|
+
baseUrl: options.llmBaseUrl ?? config3.GRAPH_EXTRACT_BASE_URL,
|
|
236240
|
+
explicitKey: options.llmApiKey ?? config3.GRAPH_EXTRACT_API_KEY,
|
|
236241
|
+
model: primaryModel,
|
|
236242
|
+
reasoningEffort: options.reasoningEffort ?? config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
236243
|
+
},
|
|
236244
|
+
{
|
|
236245
|
+
baseUrl: config3.GRAPH_EXTRACT_FALLBACK_BASE_URL,
|
|
236246
|
+
explicitKey: config3.GRAPH_EXTRACT_FALLBACK_API_KEY,
|
|
236247
|
+
model: config3.GRAPH_EXTRACT_FALLBACK_MODEL ?? primaryModel,
|
|
236248
|
+
reasoningEffort: config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
236249
|
+
},
|
|
236250
|
+
{
|
|
236251
|
+
baseUrl: config3.GRAPH_EXTRACT_FALLBACK_2_BASE_URL,
|
|
236252
|
+
explicitKey: config3.GRAPH_EXTRACT_FALLBACK_2_API_KEY,
|
|
236253
|
+
model: config3.GRAPH_EXTRACT_FALLBACK_2_MODEL,
|
|
236254
|
+
reasoningEffort: config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
236255
|
+
}
|
|
236256
|
+
];
|
|
236257
|
+
return uniqueChatProviders(slots.map((slot) => {
|
|
236258
|
+
if (!slot.baseUrl || !slot.model)
|
|
236259
|
+
return;
|
|
236260
|
+
return {
|
|
236261
|
+
baseUrl: slot.baseUrl,
|
|
236262
|
+
apiKey: resolveGraphProviderKey(slot.baseUrl, slot.explicitKey),
|
|
236263
|
+
model: slot.model,
|
|
236264
|
+
timeoutMs,
|
|
236265
|
+
reasoningEffort: slot.reasoningEffort
|
|
236266
|
+
};
|
|
236267
|
+
}));
|
|
235568
236268
|
}
|
|
235569
236269
|
var SYSTEM_PROMPT = [
|
|
235570
236270
|
"You are an entity-extraction assistant for a knowledge-base system.",
|
|
@@ -235600,39 +236300,13 @@ var extractionOutputSchema = exports_external2.object({
|
|
|
235600
236300
|
entities: exports_external2.array(exports_external2.unknown())
|
|
235601
236301
|
});
|
|
235602
236302
|
async function callEntityExtractionLLM(text4, options) {
|
|
235603
|
-
const
|
|
235604
|
-
const
|
|
235605
|
-
const primaryModel = options.llmModel ?? config3.GRAPH_EXTRACT_MODEL ?? "gpt-4o-mini";
|
|
235606
|
-
const primaryKey2 = primaryBase ? resolveGraphProviderKey(primaryBase, primaryExplicitKey) : "";
|
|
235607
|
-
const fallbackBase = config3.GRAPH_EXTRACT_FALLBACK_BASE_URL;
|
|
235608
|
-
const fallbackExplicitKey = config3.GRAPH_EXTRACT_FALLBACK_API_KEY;
|
|
235609
|
-
const fallbackModel = config3.GRAPH_EXTRACT_FALLBACK_MODEL ?? primaryModel;
|
|
235610
|
-
const fallbackKey = fallbackBase ? resolveGraphProviderKey(fallbackBase, fallbackExplicitKey) : "";
|
|
235611
|
-
const providers = [];
|
|
235612
|
-
if (primaryBase) {
|
|
235613
|
-
providers.push({
|
|
235614
|
-
baseUrl: primaryBase,
|
|
235615
|
-
apiKey: primaryKey2,
|
|
235616
|
-
model: primaryModel,
|
|
235617
|
-
timeoutMs: config3.GRAPH_EXTRACT_TIMEOUT_MS,
|
|
235618
|
-
reasoningEffort: options.reasoningEffort ?? config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
235619
|
-
});
|
|
235620
|
-
}
|
|
235621
|
-
if (fallbackBase && (fallbackBase !== primaryBase || fallbackKey !== primaryKey2 || fallbackModel !== primaryModel)) {
|
|
235622
|
-
providers.push({
|
|
235623
|
-
baseUrl: fallbackBase,
|
|
235624
|
-
apiKey: fallbackKey,
|
|
235625
|
-
model: fallbackModel,
|
|
235626
|
-
timeoutMs: config3.GRAPH_EXTRACT_TIMEOUT_MS,
|
|
235627
|
-
reasoningEffort: config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
235628
|
-
});
|
|
235629
|
-
}
|
|
235630
|
-
const [primary, fallback] = providers;
|
|
236303
|
+
const providers = graphExtractProviders(options);
|
|
236304
|
+
const [primary, ...fallbacks] = providers;
|
|
235631
236305
|
if (!primary)
|
|
235632
236306
|
return null;
|
|
235633
|
-
const
|
|
236307
|
+
const attempt = await requestStructuredChatDetailed({
|
|
235634
236308
|
primary,
|
|
235635
|
-
|
|
236309
|
+
fallbacks,
|
|
235636
236310
|
messages: [
|
|
235637
236311
|
{ role: "system", content: SYSTEM_PROMPT },
|
|
235638
236312
|
{ role: "user", content: text4 }
|
|
@@ -235641,7 +236315,9 @@ async function callEntityExtractionLLM(text4, options) {
|
|
|
235641
236315
|
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
235642
236316
|
temperature: options.temperature ?? DEFAULT_TEMPERATURE
|
|
235643
236317
|
});
|
|
235644
|
-
|
|
236318
|
+
if (!attempt.ok)
|
|
236319
|
+
throw attempt.error;
|
|
236320
|
+
return parseExtractionResponse(JSON.stringify(attempt.result.data));
|
|
235645
236321
|
}
|
|
235646
236322
|
function resolveGraphProviderKey(baseUrl, explicitKey) {
|
|
235647
236323
|
return resolveChatProviderKey(baseUrl, explicitKey, config3.OPENROUTER_API_KEY);
|
|
@@ -235835,11 +236511,11 @@ async function buildKnowledgeSummary(document2, provider) {
|
|
|
235835
236511
|
}
|
|
235836
236512
|
const output = await provider(document2);
|
|
235837
236513
|
if (!output)
|
|
235838
|
-
throw new Error("
|
|
236514
|
+
throw new Error("provider_failure");
|
|
235839
236515
|
const language = output.language.trim();
|
|
235840
236516
|
const description = output.description.trim();
|
|
235841
236517
|
if (!language || !description)
|
|
235842
|
-
throw new Error("
|
|
236518
|
+
throw new Error("permanent_validation_failure");
|
|
235843
236519
|
const seen = new Set;
|
|
235844
236520
|
const keywords = [];
|
|
235845
236521
|
for (const raw of output.keywords) {
|
|
@@ -235890,10 +236566,13 @@ async function requestKnowledgeSummary(document2) {
|
|
|
235890
236566
|
const primary = provider(config3.GRAPH_EXTRACT_BASE_URL, config3.GRAPH_EXTRACT_API_KEY, config3.GRAPH_EXTRACT_MODEL);
|
|
235891
236567
|
if (!primary)
|
|
235892
236568
|
return null;
|
|
235893
|
-
const
|
|
235894
|
-
|
|
236569
|
+
const fallbacks = [
|
|
236570
|
+
provider(config3.GRAPH_EXTRACT_FALLBACK_BASE_URL, config3.GRAPH_EXTRACT_FALLBACK_API_KEY, config3.GRAPH_EXTRACT_FALLBACK_MODEL),
|
|
236571
|
+
provider(config3.GRAPH_EXTRACT_FALLBACK_2_BASE_URL, config3.GRAPH_EXTRACT_FALLBACK_2_API_KEY, config3.GRAPH_EXTRACT_FALLBACK_2_MODEL)
|
|
236572
|
+
].filter((item) => Boolean(item));
|
|
236573
|
+
const attempt = await requestStructuredChatDetailed({
|
|
235895
236574
|
primary,
|
|
235896
|
-
|
|
236575
|
+
fallbacks,
|
|
235897
236576
|
messages: [
|
|
235898
236577
|
{ role: "system", content: systemPrompt },
|
|
235899
236578
|
{
|
|
@@ -235907,7 +236586,9 @@ ${document2.content.slice(0, 50000)}`
|
|
|
235907
236586
|
maxTokens: 768,
|
|
235908
236587
|
temperature: 0
|
|
235909
236588
|
});
|
|
235910
|
-
|
|
236589
|
+
if (!attempt.ok)
|
|
236590
|
+
throw attempt.error;
|
|
236591
|
+
return { ...attempt.result.data, model: attempt.result.model };
|
|
235911
236592
|
}
|
|
235912
236593
|
|
|
235913
236594
|
// ../../backend/src/queue/document-revision.ts
|
|
@@ -236328,7 +237009,7 @@ function providerProfile(name) {
|
|
|
236328
237009
|
};
|
|
236329
237010
|
}
|
|
236330
237011
|
function configuredEmbeddingProvider() {
|
|
236331
|
-
const model = config3.EMBEDDING_MODEL ?? config3.EMBEDDING_FALLBACK_MODEL ?? "";
|
|
237012
|
+
const model = config3.EMBEDDING_MODEL ?? config3.EMBEDDING_FALLBACK_MODEL ?? config3.EMBEDDING_FALLBACK_2_MODEL ?? "";
|
|
236332
237013
|
return {
|
|
236333
237014
|
model,
|
|
236334
237015
|
profile: embeddingProfileId(model, EMBEDDING_DIMENSIONS, "v1"),
|
|
@@ -236339,6 +237020,10 @@ function configuredEmbeddingProvider() {
|
|
|
236339
237020
|
config3.EMBEDDING_FALLBACK_BASE_URL ?? "",
|
|
236340
237021
|
config3.EMBEDDING_FALLBACK_MODEL ?? ""
|
|
236341
237022
|
],
|
|
237023
|
+
fallback_2: [
|
|
237024
|
+
config3.EMBEDDING_FALLBACK_2_BASE_URL ?? "",
|
|
237025
|
+
config3.EMBEDDING_FALLBACK_2_MODEL ?? ""
|
|
237026
|
+
],
|
|
236342
237027
|
dimensions: EMBEDDING_DIMENSIONS,
|
|
236343
237028
|
profileVersion: "v1"
|
|
236344
237029
|
})
|
|
@@ -236404,6 +237089,14 @@ function stagePatch(stage, status2) {
|
|
|
236404
237089
|
return { summarizeStatus: status2 };
|
|
236405
237090
|
return { finalizeStatus: status2 };
|
|
236406
237091
|
}
|
|
237092
|
+
function stageErrorPatch(stage, status2, errorCode) {
|
|
237093
|
+
const value = status2 === "failed" ? errorCode ?? "unknown_failure" : null;
|
|
237094
|
+
if (stage === "graph")
|
|
237095
|
+
return { graphErrorCode: value };
|
|
237096
|
+
if (stage === "summarize")
|
|
237097
|
+
return { summarizeErrorCode: value };
|
|
237098
|
+
return {};
|
|
237099
|
+
}
|
|
236407
237100
|
function pipelineStageStatus(value) {
|
|
236408
237101
|
return value === "ready_with_warnings" ? "failed" : value;
|
|
236409
237102
|
}
|
|
@@ -236433,6 +237126,7 @@ async function cancelStalePipelineRun(job2, errorCode = "stale_revision") {
|
|
|
236433
237126
|
async function setStageStatus(generationId, stage, status2, errorCode) {
|
|
236434
237127
|
await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
|
|
236435
237128
|
...stagePatch(stage, status2),
|
|
237129
|
+
...stageErrorPatch(stage, status2, errorCode),
|
|
236436
237130
|
...errorCode ? { errorCode } : {},
|
|
236437
237131
|
updatedAt: new Date
|
|
236438
237132
|
}).where(and(eq(documentPipelineRuns.generationId, generationId), ne(documentPipelineRuns.status, "cancelled"))));
|
|
@@ -236743,7 +237437,8 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
236743
237437
|
generationId: run3.generationId,
|
|
236744
237438
|
revision: run3.revision,
|
|
236745
237439
|
embeddingContextHash: run3.embeddingContextHash ?? undefined,
|
|
236746
|
-
embedStatus: pipelineStageStatus(run3.embedStatus)
|
|
237440
|
+
embedStatus: pipelineStageStatus(run3.embedStatus),
|
|
237441
|
+
summarizeStatus: pipelineStageStatus(run3.summarizeStatus)
|
|
236747
237442
|
} : null;
|
|
236748
237443
|
},
|
|
236749
237444
|
async extract(job2) {
|
|
@@ -236771,7 +237466,15 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
236771
237466
|
const data = { ...job2, stage: "summarize" };
|
|
236772
237467
|
await enqueueIfActive("summarize", "summarize", data, {
|
|
236773
237468
|
...DEFAULT_JOB_OPTIONS,
|
|
236774
|
-
jobId:
|
|
237469
|
+
jobId: summaryJobIdForPipeline(job2),
|
|
237470
|
+
priority: SOURCE_PRIORITY[job2.source]
|
|
237471
|
+
});
|
|
237472
|
+
},
|
|
237473
|
+
async enqueueFinalize(job2) {
|
|
237474
|
+
const data = { ...job2, stage: "finalize" };
|
|
237475
|
+
await enqueueIfActive("finalize", "finalize", data, {
|
|
237476
|
+
...DEFAULT_JOB_OPTIONS,
|
|
237477
|
+
jobId: `${JOB_IDS.finalize(job2.generationId, job2.workspaceId)}-graph-retry-${job2.requestedAt.replace(/\D/g, "")}`,
|
|
236775
237478
|
priority: SOURCE_PRIORITY[job2.source]
|
|
236776
237479
|
});
|
|
236777
237480
|
}
|
|
@@ -237266,10 +237969,12 @@ async function processSummaryStage(job2, dependencies) {
|
|
|
237266
237969
|
} catch (error53) {
|
|
237267
237970
|
if (!await continueIfCurrent())
|
|
237268
237971
|
return;
|
|
237269
|
-
await dependencies.setSummaryStatus(job2.generationId, "failed", error53
|
|
237972
|
+
await dependencies.setSummaryStatus(job2.generationId, "failed", pipelineErrorCode(error53, "summary_failed"));
|
|
237270
237973
|
if (!await continueIfCurrent())
|
|
237271
237974
|
return;
|
|
237272
237975
|
await dependencies.enqueueFinalize(job2);
|
|
237976
|
+
if (error53 instanceof PipelineProviderError && error53.retryable)
|
|
237977
|
+
throw error53;
|
|
237273
237978
|
return;
|
|
237274
237979
|
}
|
|
237275
237980
|
if (!await continueIfCurrent())
|
|
@@ -237284,7 +237989,7 @@ async function processGraphStageFailure(job2, error53, dependencies) {
|
|
|
237284
237989
|
}
|
|
237285
237990
|
if (await dependencies.isCancelled?.(job2))
|
|
237286
237991
|
throw error53;
|
|
237287
|
-
await dependencies.setGraphStatus(job2.generationId, "failed", error53
|
|
237992
|
+
await dependencies.setGraphStatus(job2.generationId, "failed", pipelineErrorCode(error53, "graph_failed"));
|
|
237288
237993
|
if (!await dependencies.isCancelled?.(job2)) {
|
|
237289
237994
|
await dependencies.enqueueSummarize(job2);
|
|
237290
237995
|
}
|
|
@@ -237318,6 +238023,18 @@ function createGraphWorker(deps) {
|
|
|
237318
238023
|
const run3 = await deps.getRun(job2);
|
|
237319
238024
|
if (!run3)
|
|
237320
238025
|
throw new Error("Pipeline run not found");
|
|
238026
|
+
const continuePipeline = async () => {
|
|
238027
|
+
if (await deps.isCancelled?.(job2))
|
|
238028
|
+
return;
|
|
238029
|
+
if (run3.summarizeStatus === "ready" || run3.summarizeStatus === "skipped") {
|
|
238030
|
+
if (deps.enqueueFinalize)
|
|
238031
|
+
await deps.enqueueFinalize(job2);
|
|
238032
|
+
else
|
|
238033
|
+
await deps.enqueueSummarize(job2);
|
|
238034
|
+
return;
|
|
238035
|
+
}
|
|
238036
|
+
await deps.enqueueSummarize(job2);
|
|
238037
|
+
};
|
|
237321
238038
|
if (run3.ownerId !== job2.ownerId || run3.documentId !== job2.documentId) {
|
|
237322
238039
|
throw new Error("Pipeline owner mismatch");
|
|
237323
238040
|
}
|
|
@@ -237335,8 +238052,7 @@ function createGraphWorker(deps) {
|
|
|
237335
238052
|
if (await deps.isCancelled?.(job2))
|
|
237336
238053
|
return;
|
|
237337
238054
|
await deps.setGraphStatus(job2.generationId, "skipped", "embedding_not_ready");
|
|
237338
|
-
|
|
237339
|
-
await deps.enqueueSummarize(job2);
|
|
238055
|
+
await continuePipeline();
|
|
237340
238056
|
return;
|
|
237341
238057
|
}
|
|
237342
238058
|
if (await deps.isCancelled?.(job2))
|
|
@@ -237356,15 +238072,27 @@ function createGraphWorker(deps) {
|
|
|
237356
238072
|
return;
|
|
237357
238073
|
}
|
|
237358
238074
|
if (outcome.status === "unavailable" || outcome.status === "failed") {
|
|
238075
|
+
if (outcome.status === "failed" && [
|
|
238076
|
+
"provider_failure",
|
|
238077
|
+
"provider_timeout",
|
|
238078
|
+
"invalid_provider_response"
|
|
238079
|
+
].includes(outcome.warning)) {
|
|
238080
|
+
throw new PipelineProviderError(outcome.warning, true);
|
|
238081
|
+
}
|
|
237359
238082
|
await deps.setGraphStatus(job2.generationId, "failed", outcome.warning);
|
|
237360
|
-
|
|
237361
|
-
await deps.enqueueSummarize(job2);
|
|
238083
|
+
await continuePipeline();
|
|
237362
238084
|
return;
|
|
237363
238085
|
}
|
|
237364
238086
|
await deps.setGraphStatus(job2.generationId, "ready");
|
|
237365
|
-
|
|
237366
|
-
await deps.enqueueSummarize(job2);
|
|
238087
|
+
await continuePipeline();
|
|
237367
238088
|
} catch (error53) {
|
|
238089
|
+
if (error53 instanceof PipelineProviderError) {
|
|
238090
|
+
await deps.setGraphStatus(job2.generationId, "failed", error53.code);
|
|
238091
|
+
await continuePipeline();
|
|
238092
|
+
if (error53.retryable)
|
|
238093
|
+
throw error53;
|
|
238094
|
+
return;
|
|
238095
|
+
}
|
|
237368
238096
|
await processGraphStageFailure(job2, error53, deps);
|
|
237369
238097
|
}
|
|
237370
238098
|
};
|
|
@@ -237571,7 +238299,7 @@ var swaggerConfig = {
|
|
|
237571
238299
|
},
|
|
237572
238300
|
info: {
|
|
237573
238301
|
title: "DocsMint API",
|
|
237574
|
-
version: "0.
|
|
238302
|
+
version: "0.8.2",
|
|
237575
238303
|
description: "Self-hosted AI-native knowledge workspace and installable PWA with hybrid search, GraphRAG, REST, SDK, CLI, and MCP access for people and AI agents.",
|
|
237576
238304
|
contact: { name: "HiAi-gg", url: "https://github.com/HiAi-gg/docsmint" },
|
|
237577
238305
|
license: {
|
|
@@ -237632,7 +238360,7 @@ app.listen({
|
|
|
237632
238360
|
idleTimeout: 30,
|
|
237633
238361
|
maxRequestBodySize: MAX_BODY_SIZE_BYTES
|
|
237634
238362
|
});
|
|
237635
|
-
logger3.info({ port: config3.API_PORT }, "
|
|
238363
|
+
logger3.info({ port: config3.API_PORT }, "DocsMint API started");
|
|
237636
238364
|
var stopDocsMintApi = async () => {
|
|
237637
238365
|
logger3.info("Shutting down...");
|
|
237638
238366
|
attachmentUploadCleanup.stop();
|