@hiai-gg/docsmint 0.5.10 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +39 -8
- package/README.md +90 -46
- package/dist/backend/index.js +797 -136
- package/dist/backend-account-runtime-cleanup.js +17 -0
- package/dist/backend-api-key-facade.js +17 -0
- package/dist/backend-pipeline-cancellation.js +17 -0
- package/dist/client.d.ts +6 -1
- package/dist/client.js +11 -0
- package/dist/frontend/HiAiEditor-DSVG3GNd.js +2986 -0
- package/dist/frontend/SettingsDialog-BDNgf5rv.js +969 -0
- package/dist/frontend/ShareDialog-2Mty7hm4.js +258 -0
- package/dist/frontend/{Sidebar-CzcLd6v5.js → Sidebar-DX08utdV.js} +2 -2
- package/dist/frontend/app-shell.js +27 -26
- package/dist/frontend/components/editor/compact-editor.d.ts +3 -0
- package/dist/frontend/components/editor/compact-editor.js +36 -0
- package/dist/frontend/components/editor/document-editor.js +2 -2986
- package/dist/frontend/components/settings.js +1 -1
- package/dist/frontend/components/share-dialog.d.ts +2 -1
- package/dist/frontend/components/share-dialog.js +1 -1
- package/dist/frontend/components/sidebar.js +1 -1
- package/dist/frontend/dashboard.js +1 -1
- package/dist/frontend/editor-preferences.d.ts +4 -0
- package/dist/frontend/editor-preferences.js +2 -0
- package/dist/frontend/editor-preferences.svelte-Bt_A9k7n.js +61 -0
- package/dist/frontend/frontend.css +1 -1
- package/dist/frontend-ssr/app-shell.js +3 -2
- package/dist/frontend-ssr/assets/HiAiEditor-CUOJmKzT.js +2358 -0
- package/dist/frontend-ssr/assets/{SettingsDialog-DVp0OxPO.js → SettingsDialog-BjJIiVLc.js} +2 -64
- package/dist/frontend-ssr/assets/{ShareDialog-BZd28sg-.js → ShareDialog-C9C-XTm4.js} +1 -1
- package/dist/frontend-ssr/assets/{Sidebar-CqsMw0Hd.js → Sidebar-C2sUfGNk.js} +2 -2
- package/dist/frontend-ssr/assets/editor-preferences.svelte-iI5jJM1x.js +65 -0
- package/dist/frontend-ssr/components/editor/compact-editor.js +19 -0
- package/dist/frontend-ssr/components/editor/document-editor.js +1 -2357
- package/dist/frontend-ssr/components/settings.js +1 -1
- package/dist/frontend-ssr/components/share-dialog.js +1 -1
- package/dist/frontend-ssr/components/sidebar.js +1 -1
- package/dist/frontend-ssr/dashboard.js +1 -1
- package/dist/frontend-ssr/editor-preferences.js +2 -0
- package/dist/lifecycle-runtime.js +16 -0
- package/dist/types.d.ts +31 -0
- package/package.json +20 -2
- package/packages/cli/src/index.ts +1 -1
- package/packages/db/src/index.ts +16 -7
- package/packages/db/src/schema.ts +34 -0
- package/packages/mcp-server/src/capabilities.ts +165 -0
- package/packages/mcp-server/src/client.ts +191 -138
- package/packages/mcp-server/src/index.ts +4 -104
- package/packages/mcp-server/src/server.ts +82 -0
- package/packages/mcp-server/src/tools/create-document.ts +26 -19
- package/packages/mcp-server/src/tools/create-folder.ts +19 -12
- package/packages/mcp-server/src/tools/update-document.ts +31 -17
- package/skills/docsmint-document-manager/SKILL.md +33 -0
- package/dist/frontend/SettingsDialog-CyZSOjLE.js +0 -1024
- package/dist/frontend/ShareDialog-BC-Jl6v1.js +0 -257
package/dist/backend/index.js
CHANGED
|
@@ -42744,6 +42744,7 @@ __export(exports_schema, {
|
|
|
42744
42744
|
documentRelations: () => documentRelations,
|
|
42745
42745
|
documentPipelineRuns: () => documentPipelineRuns,
|
|
42746
42746
|
documentPipelineBatches: () => documentPipelineBatches,
|
|
42747
|
+
documentKnowledgeSummaries: () => documentKnowledgeSummaries,
|
|
42747
42748
|
documentEmbeddings: () => documentEmbeddings,
|
|
42748
42749
|
documentEmbeddingRelations: () => documentEmbeddingRelations,
|
|
42749
42750
|
documentCreateOperations: () => documentCreateOperations,
|
|
@@ -42756,7 +42757,7 @@ __export(exports_schema, {
|
|
|
42756
42757
|
apiKeyRelations: () => apiKeyRelations,
|
|
42757
42758
|
accounts: () => accounts
|
|
42758
42759
|
});
|
|
42759
|
-
var vector4, tsvector, documentVisibilityEnum, shareRoleEnum, shareAccessModeEnum, shareGrantStatusEnum, embeddingStatusEnum, pipelineStageEnum, pipelineStatusEnum, lifecycleOperationKindEnum, lifecycleOperationStatusEnum, users, sessions, accounts, verifications, folders, folderRelations, documents, documentRelations, documentPipelineRuns, documentPipelineBatches, tags, tagRelations, categories, categoryRelations, documentTags, documentTagRelations, shareLinks, shareLinkRelations, guestAccess, guestAccessRelations, attachments, attachmentRelations, versions, documentEmbeddings, documentEmbeddingRelations, apiKeys, apiKeyRelations, auditLog, lifecycleOperations, documentCreateOperations, versionRelations;
|
|
42760
|
+
var vector4, tsvector, documentVisibilityEnum, shareRoleEnum, shareAccessModeEnum, shareGrantStatusEnum, embeddingStatusEnum, pipelineStageEnum, pipelineStatusEnum, lifecycleOperationKindEnum, lifecycleOperationStatusEnum, users, sessions, accounts, verifications, folders, folderRelations, documents, documentRelations, documentPipelineRuns, documentPipelineBatches, documentKnowledgeSummaries, tags, tagRelations, categories, categoryRelations, documentTags, documentTagRelations, shareLinks, shareLinkRelations, guestAccess, guestAccessRelations, attachments, attachmentRelations, versions, documentEmbeddings, documentEmbeddingRelations, apiKeys, apiKeyRelations, auditLog, lifecycleOperations, documentCreateOperations, versionRelations;
|
|
42760
42761
|
var init_schema2 = __esm(() => {
|
|
42761
42762
|
init_pg_core();
|
|
42762
42763
|
init_drizzle_orm();
|
|
@@ -43010,6 +43011,22 @@ var init_schema2 = __esm(() => {
|
|
|
43010
43011
|
index("document_pipeline_batches_stage_status_available_idx").on(table3.stage, table3.status, table3.availableAt),
|
|
43011
43012
|
index("document_pipeline_batches_document_id_idx").on(table3.documentId)
|
|
43012
43013
|
]);
|
|
43014
|
+
documentKnowledgeSummaries = pgTable("document_knowledge_summaries", {
|
|
43015
|
+
id: uuid().primaryKey().defaultRandom(),
|
|
43016
|
+
document_id: uuid().notNull().references(() => documents.id, { onDelete: "cascade" }),
|
|
43017
|
+
owner_id: uuid().notNull().references(() => users.id, { onDelete: "cascade" }),
|
|
43018
|
+
workspace_id: text(),
|
|
43019
|
+
generation_id: uuid().notNull(),
|
|
43020
|
+
revision: text().notNull(),
|
|
43021
|
+
language: text().notNull(),
|
|
43022
|
+
description: text().notNull(),
|
|
43023
|
+
keywords: text().array().notNull().default(sql`ARRAY[]::text[]`),
|
|
43024
|
+
created_at: timestamp().defaultNow().notNull(),
|
|
43025
|
+
updated_at: timestamp().defaultNow().notNull()
|
|
43026
|
+
}, (table3) => [
|
|
43027
|
+
uniqueIndex("document_knowledge_summaries_document_generation_idx").on(table3.document_id, table3.generation_id),
|
|
43028
|
+
index("document_knowledge_summaries_owner_document_idx").on(table3.owner_id, table3.document_id)
|
|
43029
|
+
]);
|
|
43013
43030
|
tags = pgTable("tags", {
|
|
43014
43031
|
id: uuid("id").primaryKey().defaultRandom(),
|
|
43015
43032
|
ownerId: uuid("owner_id").notNull().references(() => users.id, { onDelete: "cascade" }),
|
|
@@ -173203,6 +173220,8 @@ async function validateApiKey(key) {
|
|
|
173203
173220
|
scopes
|
|
173204
173221
|
};
|
|
173205
173222
|
}
|
|
173223
|
+
// ../db/src/document-writer.ts
|
|
173224
|
+
init_schema2();
|
|
173206
173225
|
|
|
173207
173226
|
// ../db/src/index.ts
|
|
173208
173227
|
init_schema2();
|
|
@@ -194094,6 +194113,7 @@ function contentAccessForExternalContext(ctx) {
|
|
|
194094
194113
|
permissions.add("write");
|
|
194095
194114
|
} else if (ctx.actorRole === "editor") {
|
|
194096
194115
|
permissions.add("edit");
|
|
194116
|
+
permissions.add("write");
|
|
194097
194117
|
}
|
|
194098
194118
|
return {
|
|
194099
194119
|
principal: { kind: "session", userId: ctx.userId },
|
|
@@ -194144,6 +194164,9 @@ function canAccessContent(access, action) {
|
|
|
194144
194164
|
return access.permissions.has(action);
|
|
194145
194165
|
return !access.restricted || access.permissions.has(action);
|
|
194146
194166
|
}
|
|
194167
|
+
function canManageCategories(access) {
|
|
194168
|
+
return !access.restricted && canAccessContent(access, "write");
|
|
194169
|
+
}
|
|
194147
194170
|
function effectiveDocumentCategory(row) {
|
|
194148
194171
|
return row.categoryId ?? row.folderCategoryId ?? null;
|
|
194149
194172
|
}
|
|
@@ -194237,7 +194260,8 @@ var enqueueDocumentPipelineSchema = exports_external2.object({
|
|
|
194237
194260
|
workspaceId: exports_external2.string().min(1).optional(),
|
|
194238
194261
|
revision: exports_external2.string().min(1),
|
|
194239
194262
|
source: pipelineSourceSchema,
|
|
194240
|
-
requestedAt: exports_external2.iso.datetime().optional()
|
|
194263
|
+
requestedAt: exports_external2.iso.datetime().optional(),
|
|
194264
|
+
forceNewGeneration: exports_external2.boolean().optional()
|
|
194241
194265
|
});
|
|
194242
194266
|
var JOB_IDS = {
|
|
194243
194267
|
prepare: (documentId, generationId, workspaceId) => `prepare-${documentId}-${generationId}${workspaceId ? `-${workspaceId}` : ""}`,
|
|
@@ -194266,7 +194290,14 @@ var postgresRunStore = {
|
|
|
194266
194290
|
const [document2] = await tx.select({ id: documents.id }).from(documents).where(and(eq(documents.id, input.documentId), ownerBoundary)).limit(1);
|
|
194267
194291
|
if (!document2)
|
|
194268
194292
|
throw new Error("Document not found for pipeline owner");
|
|
194269
|
-
|
|
194293
|
+
if (input.forceNewGeneration) {
|
|
194294
|
+
await tx.update(documentPipelineRuns).set({
|
|
194295
|
+
status: "cancelled",
|
|
194296
|
+
errorCode: "superseded_by_reindex",
|
|
194297
|
+
updatedAt: new Date
|
|
194298
|
+
}).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES])));
|
|
194299
|
+
}
|
|
194300
|
+
const [existing] = input.forceNewGeneration ? [] : await tx.select({ generationId: documentPipelineRuns.generationId }).from(documentPipelineRuns).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, eq(documentPipelineRuns.revision, input.revision), inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES]))).limit(1);
|
|
194270
194301
|
if (existing)
|
|
194271
194302
|
return { run: existing, created: false };
|
|
194272
194303
|
const [created] = await tx.insert(documentPipelineRuns).values({
|
|
@@ -194302,7 +194333,8 @@ async function enqueueDocumentPipeline(input, dependencies) {
|
|
|
194302
194333
|
const { run, created } = await deps.runs.findOrCreate({
|
|
194303
194334
|
...parsed,
|
|
194304
194335
|
requestedAt,
|
|
194305
|
-
generationId: proposedGenerationId
|
|
194336
|
+
generationId: proposedGenerationId,
|
|
194337
|
+
forceNewGeneration: parsed.forceNewGeneration
|
|
194306
194338
|
});
|
|
194307
194339
|
if (created) {
|
|
194308
194340
|
const job2 = {
|
|
@@ -194373,10 +194405,10 @@ ${content52}`;
|
|
|
194373
194405
|
init_logger3();
|
|
194374
194406
|
init_redis();
|
|
194375
194407
|
var LEGACY_EMBEDDING_QUEUE_KEY = "hiai-docs:embedding-queue";
|
|
194376
|
-
async function enqueueEmbedding(documentId, source = "interactive") {
|
|
194408
|
+
async function enqueueEmbedding(documentId, source = "interactive", workspaceId, options = {}) {
|
|
194377
194409
|
let pipelineInput;
|
|
194378
194410
|
try {
|
|
194379
|
-
await withTenant(adminTenantContext(ZERO_UUID), async (tx) => {
|
|
194411
|
+
await withTenant({ ...adminTenantContext(ZERO_UUID), workspaceId }, async (tx) => {
|
|
194380
194412
|
const rows = await tx.select({
|
|
194381
194413
|
active: documents.activeEmbeddingGeneration,
|
|
194382
194414
|
ownerId: documents.ownerId,
|
|
@@ -194387,7 +194419,7 @@ async function enqueueEmbedding(documentId, source = "interactive") {
|
|
|
194387
194419
|
if (!document2)
|
|
194388
194420
|
return;
|
|
194389
194421
|
const revision = contentHash(document2.title, document2.content ?? "");
|
|
194390
|
-
pipelineInput = { ownerId: document2.ownerId, revision };
|
|
194422
|
+
pipelineInput = { ownerId: document2.ownerId, revision, workspaceId };
|
|
194391
194423
|
const activeGeneration = document2.active;
|
|
194392
194424
|
if (activeGeneration) {
|
|
194393
194425
|
await tx.update(documents).set({
|
|
@@ -194410,8 +194442,10 @@ async function enqueueEmbedding(documentId, source = "interactive") {
|
|
|
194410
194442
|
await enqueueDocumentPipeline({
|
|
194411
194443
|
documentId,
|
|
194412
194444
|
ownerId: pipelineInput.ownerId,
|
|
194445
|
+
workspaceId: pipelineInput.workspaceId,
|
|
194413
194446
|
revision: pipelineInput.revision,
|
|
194414
|
-
source
|
|
194447
|
+
source,
|
|
194448
|
+
forceNewGeneration: options.forceNewGeneration
|
|
194415
194449
|
});
|
|
194416
194450
|
return true;
|
|
194417
194451
|
} catch (err) {
|
|
@@ -194449,7 +194483,7 @@ async function enqueueReembed(docIds, workspaceId) {
|
|
|
194449
194483
|
let pushed = 0;
|
|
194450
194484
|
for (const id2 of unique) {
|
|
194451
194485
|
if (await claimEnqueueSlot(id2, workspaceId)) {
|
|
194452
|
-
enqueueEmbedding(id2);
|
|
194486
|
+
enqueueEmbedding(id2, "interactive", workspaceId);
|
|
194453
194487
|
pushed += 1;
|
|
194454
194488
|
}
|
|
194455
194489
|
}
|
|
@@ -196359,9 +196393,6 @@ init_schema2();
|
|
|
196359
196393
|
init_drizzle_orm();
|
|
196360
196394
|
init_zod();
|
|
196361
196395
|
init_logger3();
|
|
196362
|
-
function isCategoryManager(principal) {
|
|
196363
|
-
return principal?.kind === "session" || principal?.kind === "operator";
|
|
196364
|
-
}
|
|
196365
196396
|
var createCategorySchema = exports_external2.object({
|
|
196366
196397
|
name: exports_external2.string().trim().min(1).max(255),
|
|
196367
196398
|
apiMode: exports_external2.enum(["unavailable", "global", "category"]).optional(),
|
|
@@ -196499,9 +196530,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
|
|
|
196499
196530
|
set2.status = 401;
|
|
196500
196531
|
return { error: "Unauthorized" };
|
|
196501
196532
|
}
|
|
196502
|
-
if (!
|
|
196533
|
+
if (!canManageCategories(access)) {
|
|
196503
196534
|
set2.status = 403;
|
|
196504
|
-
return { error: "
|
|
196535
|
+
return { error: "Full workspace write access required" };
|
|
196505
196536
|
}
|
|
196506
196537
|
const userId = ctx.userId;
|
|
196507
196538
|
const parsed = createCategorySchema.safeParse(await request.json());
|
|
@@ -196551,9 +196582,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
|
|
|
196551
196582
|
set2.status = 401;
|
|
196552
196583
|
return { error: "Unauthorized" };
|
|
196553
196584
|
}
|
|
196554
|
-
if (!
|
|
196585
|
+
if (!canManageCategories(access)) {
|
|
196555
196586
|
set2.status = 403;
|
|
196556
|
-
return { error: "
|
|
196587
|
+
return { error: "Full workspace write access required" };
|
|
196557
196588
|
}
|
|
196558
196589
|
const userId = ctx.userId;
|
|
196559
196590
|
const parsed = updateCategorySchema.safeParse(await request.json());
|
|
@@ -196622,9 +196653,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
|
|
|
196622
196653
|
set2.status = 401;
|
|
196623
196654
|
return { error: "Unauthorized" };
|
|
196624
196655
|
}
|
|
196625
|
-
if (!
|
|
196656
|
+
if (!canManageCategories(access)) {
|
|
196626
196657
|
set2.status = 403;
|
|
196627
|
-
return { error: "
|
|
196658
|
+
return { error: "Full workspace write access required" };
|
|
196628
196659
|
}
|
|
196629
196660
|
const userId = ctx.userId;
|
|
196630
196661
|
try {
|
|
@@ -227119,8 +227150,11 @@ var listQuerySchema2 = exports_external2.object({
|
|
|
227119
227150
|
var cursorListQuerySchema = exports_external2.object({
|
|
227120
227151
|
categoryId: exports_external2.string().uuid().optional(),
|
|
227121
227152
|
cursor: exports_external2.string().min(1).optional(),
|
|
227122
|
-
limit: exports_external2.coerce.number().int().min(1).max(100).default(50)
|
|
227153
|
+
limit: exports_external2.coerce.number().int().min(1).max(100).default(50),
|
|
227154
|
+
sortBy: exports_external2.enum(["title", "category", "folder", "updated"]).default("updated"),
|
|
227155
|
+
sortOrder: exports_external2.enum(["asc", "desc"]).default("desc")
|
|
227123
227156
|
});
|
|
227157
|
+
var documentIdParamsSchema = exports_external2.object({ id: exports_external2.string().uuid() });
|
|
227124
227158
|
function cursorScopeHash(input) {
|
|
227125
227159
|
return Buffer.from(new Bun.CryptoHasher("sha256").update(input).digest("hex"), "utf8").toString("base64url");
|
|
227126
227160
|
}
|
|
@@ -227134,7 +227168,7 @@ function decodeDocumentCursor(value, expectedScopeHash) {
|
|
|
227134
227168
|
if (!cursor || typeof cursor !== "object")
|
|
227135
227169
|
throw new Error("Malformed cursor");
|
|
227136
227170
|
const candidate = cursor;
|
|
227137
|
-
if (candidate.v !==
|
|
227171
|
+
if (candidate.v !== 2 || !["title", "category", "folder", "updated"].includes(String(candidate.sortBy)) || !["asc", "desc"].includes(String(candidate.sortOrder)) || candidate.value !== null && typeof candidate.value !== "string" || typeof candidate.id !== "string" || typeof candidate.scopeHash !== "string" || candidate.scopeHash !== expectedScopeHash) {
|
|
227138
227172
|
throw new Error("Cursor does not match this listing scope");
|
|
227139
227173
|
}
|
|
227140
227174
|
return candidate;
|
|
@@ -227378,12 +227412,20 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
227378
227412
|
set2.status = 400;
|
|
227379
227413
|
return { error: "Invalid query", details: parsed.error.flatten() };
|
|
227380
227414
|
}
|
|
227381
|
-
const {
|
|
227415
|
+
const {
|
|
227416
|
+
categoryId,
|
|
227417
|
+
cursor: encodedCursor,
|
|
227418
|
+
limit,
|
|
227419
|
+
sortBy,
|
|
227420
|
+
sortOrder
|
|
227421
|
+
} = parsed.data;
|
|
227382
227422
|
const effectiveCategoryId = access.restricted ? access.categoryId ?? categoryId : categoryId;
|
|
227383
227423
|
const scopeHash = cursorScopeHash(JSON.stringify({
|
|
227384
227424
|
workspaceId: ctx.workspaceId ?? null,
|
|
227385
227425
|
actorUserId: access.userId,
|
|
227386
|
-
categoryId: effectiveCategoryId ?? null
|
|
227426
|
+
categoryId: effectiveCategoryId ?? null,
|
|
227427
|
+
sortBy,
|
|
227428
|
+
sortOrder
|
|
227387
227429
|
}));
|
|
227388
227430
|
let cursor;
|
|
227389
227431
|
if (encodedCursor) {
|
|
@@ -227396,31 +227438,54 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
227396
227438
|
};
|
|
227397
227439
|
}
|
|
227398
227440
|
}
|
|
227441
|
+
const sortValue = sortBy === "title" ? sql`lower(${documents.title})` : sortBy === "category" ? sql`lower(${categories.name})` : sortBy === "folder" ? sql`lower(${folders.name})` : sql`${documents.updatedAt}`;
|
|
227399
227442
|
const conditions2 = [
|
|
227400
227443
|
tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx),
|
|
227401
227444
|
isNull(documents.deletedAt),
|
|
227402
|
-
...effectiveCategoryId ? [eq(documents.categoryId, effectiveCategoryId)] : []
|
|
227403
|
-
...cursor ? [
|
|
227404
|
-
or(lt(documents.updatedAt, new Date(cursor.updatedAt)), and(eq(documents.updatedAt, new Date(cursor.updatedAt)), lt(documents.id, cursor.id)))
|
|
227405
|
-
] : []
|
|
227445
|
+
...effectiveCategoryId ? [eq(documents.categoryId, effectiveCategoryId)] : []
|
|
227406
227446
|
];
|
|
227447
|
+
if (cursor) {
|
|
227448
|
+
const idAfter = sortOrder === "asc" ? gt(documents.id, cursor.id) : lt(documents.id, cursor.id);
|
|
227449
|
+
if (sortBy === "updated" && cursor.value) {
|
|
227450
|
+
const cursorDate = new Date(cursor.value);
|
|
227451
|
+
const valueAfter = sortOrder === "asc" ? gt(documents.updatedAt, cursorDate) : lt(documents.updatedAt, cursorDate);
|
|
227452
|
+
const cursorCondition = or(valueAfter, and(eq(documents.updatedAt, cursorDate), idAfter));
|
|
227453
|
+
if (cursorCondition)
|
|
227454
|
+
conditions2.push(cursorCondition);
|
|
227455
|
+
} else if (cursor.value === null) {
|
|
227456
|
+
const cursorCondition = and(isNull(sortValue), idAfter);
|
|
227457
|
+
if (cursorCondition)
|
|
227458
|
+
conditions2.push(cursorCondition);
|
|
227459
|
+
} else {
|
|
227460
|
+
const valueAfter = sortOrder === "asc" ? gt(sortValue, cursor.value) : lt(sortValue, cursor.value);
|
|
227461
|
+
const cursorCondition = or(valueAfter, and(eq(sortValue, cursor.value), idAfter), isNull(sortValue));
|
|
227462
|
+
if (cursorCondition)
|
|
227463
|
+
conditions2.push(cursorCondition);
|
|
227464
|
+
}
|
|
227465
|
+
}
|
|
227407
227466
|
try {
|
|
227408
227467
|
const rows = await withTenant(ctx, (tx) => tx.select({
|
|
227409
227468
|
id: documents.id,
|
|
227410
227469
|
title: documents.title,
|
|
227411
227470
|
content: sql`LEFT(${documents.content}, 200)`.as("content"),
|
|
227412
227471
|
folderId: documents.folderId,
|
|
227472
|
+
folderName: folders.name,
|
|
227473
|
+
categoryId: documents.categoryId,
|
|
227474
|
+
categoryName: categories.name,
|
|
227413
227475
|
createdAt: documents.createdAt,
|
|
227414
227476
|
updatedAt: documents.updatedAt
|
|
227415
|
-
}).from(documents).where(and(...conditions2)).orderBy(desc(documents.
|
|
227477
|
+
}).from(documents).leftJoin(folders, eq(folders.id, documents.folderId)).leftJoin(categories, eq(categories.id, documents.categoryId)).where(and(...conditions2)).orderBy(sortOrder === "asc" ? sql`${sortValue} asc nulls last` : sql`${sortValue} desc nulls last`, sortOrder === "asc" ? asc(documents.id) : desc(documents.id)).limit(limit + 1));
|
|
227416
227478
|
const hasMore = rows.length > limit;
|
|
227417
227479
|
const page = rows.slice(0, limit);
|
|
227418
227480
|
const last2 = page.at(-1);
|
|
227481
|
+
const lastValue = last2 ? sortBy === "title" ? last2.title.toLocaleLowerCase() : sortBy === "category" ? last2.categoryName?.toLocaleLowerCase() ?? null : sortBy === "folder" ? last2.folderName?.toLocaleLowerCase() ?? null : new Date(last2.updatedAt).toISOString() : null;
|
|
227419
227482
|
return {
|
|
227420
227483
|
items: await withTags(ctx, page),
|
|
227421
227484
|
nextCursor: hasMore && last2 ? encodeDocumentCursor({
|
|
227422
|
-
v:
|
|
227423
|
-
|
|
227485
|
+
v: 2,
|
|
227486
|
+
sortBy,
|
|
227487
|
+
sortOrder,
|
|
227488
|
+
value: lastValue,
|
|
227424
227489
|
id: last2.id,
|
|
227425
227490
|
scopeHash
|
|
227426
227491
|
}) : null
|
|
@@ -227608,6 +227673,180 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
227608
227673
|
set2.status = 500;
|
|
227609
227674
|
return { error: "Failed to load pipeline progress" };
|
|
227610
227675
|
}
|
|
227676
|
+
}).get("/documents/:id/knowledge-summary", async ({ params: params2, set: set2, request }) => {
|
|
227677
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
227678
|
+
if (!parsedParams.success) {
|
|
227679
|
+
set2.status = 400;
|
|
227680
|
+
return { error: "Invalid document id" };
|
|
227681
|
+
}
|
|
227682
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227683
|
+
const rl = await documentRateLimiter(ip, request);
|
|
227684
|
+
if (!rl.allowed) {
|
|
227685
|
+
set2.status = 429;
|
|
227686
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
227687
|
+
return { error: "Too many requests" };
|
|
227688
|
+
}
|
|
227689
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
227690
|
+
const access = await resolveContentAccess(request);
|
|
227691
|
+
const ctx = access.ctx;
|
|
227692
|
+
if (ctx.role === "none") {
|
|
227693
|
+
set2.status = 401;
|
|
227694
|
+
return { error: "Unauthorized" };
|
|
227695
|
+
}
|
|
227696
|
+
if (!canAccessContent(access, "read")) {
|
|
227697
|
+
set2.status = 403;
|
|
227698
|
+
return { error: "Forbidden" };
|
|
227699
|
+
}
|
|
227700
|
+
const [summary] = await withTenant(ctx, (tx) => tx.select({
|
|
227701
|
+
documentId: documentKnowledgeSummaries.document_id,
|
|
227702
|
+
generationId: documentKnowledgeSummaries.generation_id,
|
|
227703
|
+
revision: documentKnowledgeSummaries.revision,
|
|
227704
|
+
language: documentKnowledgeSummaries.language,
|
|
227705
|
+
description: documentKnowledgeSummaries.description,
|
|
227706
|
+
keywords: documentKnowledgeSummaries.keywords,
|
|
227707
|
+
createdAt: documentKnowledgeSummaries.created_at,
|
|
227708
|
+
updatedAt: documentKnowledgeSummaries.updated_at
|
|
227709
|
+
}).from(documentKnowledgeSummaries).innerJoin(documents, eq(documents.id, documentKnowledgeSummaries.document_id)).where(and(eq(documentKnowledgeSummaries.document_id, parsedParams.data.id), eq(documentKnowledgeSummaries.generation_id, documents.activeEmbeddingGeneration), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1));
|
|
227710
|
+
if (!summary) {
|
|
227711
|
+
set2.status = 404;
|
|
227712
|
+
return { error: "Knowledge summary not found" };
|
|
227713
|
+
}
|
|
227714
|
+
return summary;
|
|
227715
|
+
}).get("/documents/:id/index-status", async ({ params: params2, set: set2, request }) => {
|
|
227716
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
227717
|
+
if (!parsedParams.success) {
|
|
227718
|
+
set2.status = 400;
|
|
227719
|
+
return { error: "Invalid document id" };
|
|
227720
|
+
}
|
|
227721
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227722
|
+
const rl = await documentRateLimiter(ip, request);
|
|
227723
|
+
if (!rl.allowed) {
|
|
227724
|
+
set2.status = 429;
|
|
227725
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
227726
|
+
return { error: "Too many requests" };
|
|
227727
|
+
}
|
|
227728
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
227729
|
+
const access = await resolveContentAccess(request);
|
|
227730
|
+
const ctx = access.ctx;
|
|
227731
|
+
if (ctx.role === "none") {
|
|
227732
|
+
set2.status = 401;
|
|
227733
|
+
return { error: "Unauthorized" };
|
|
227734
|
+
}
|
|
227735
|
+
if (!canAccessContent(access, "read")) {
|
|
227736
|
+
set2.status = 403;
|
|
227737
|
+
return { error: "Forbidden" };
|
|
227738
|
+
}
|
|
227739
|
+
const result = await withTenant(ctx, async (tx) => {
|
|
227740
|
+
const [document2] = await tx.select({
|
|
227741
|
+
id: documents.id,
|
|
227742
|
+
embeddingStatus: documents.embeddingStatus,
|
|
227743
|
+
activeGenerationId: documents.activeEmbeddingGeneration,
|
|
227744
|
+
pendingGenerationId: documents.pendingEmbeddingGeneration,
|
|
227745
|
+
embeddingProfile: documents.embeddingProfile,
|
|
227746
|
+
embeddingErrorCode: documents.embeddingErrorCode,
|
|
227747
|
+
embeddingUpdatedAt: documents.embeddingUpdatedAt
|
|
227748
|
+
}).from(documents).where(and(eq(documents.id, parsedParams.data.id), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1);
|
|
227749
|
+
if (!document2)
|
|
227750
|
+
return null;
|
|
227751
|
+
const active = document2.activeGenerationId ? await tx.select({ id: documentEmbeddings.id }).from(documentEmbeddings).where(and(eq(documentEmbeddings.documentId, document2.id), eq(documentEmbeddings.generationId, document2.activeGenerationId), eq(documentEmbeddings.isValid, true), eq(documentEmbeddings.embeddingDimensions, 1024), eq(documentEmbeddings.embeddingProfile, document2.embeddingProfile ?? ""))).limit(1) : [];
|
|
227752
|
+
const [run5] = await tx.select({
|
|
227753
|
+
documentId: documentPipelineRuns.documentId,
|
|
227754
|
+
generationId: documentPipelineRuns.generationId,
|
|
227755
|
+
revision: documentPipelineRuns.revision,
|
|
227756
|
+
status: documentPipelineRuns.status,
|
|
227757
|
+
prepareStatus: documentPipelineRuns.prepareStatus,
|
|
227758
|
+
embedStatus: documentPipelineRuns.embedStatus,
|
|
227759
|
+
graphStatus: documentPipelineRuns.graphStatus,
|
|
227760
|
+
summarizeStatus: documentPipelineRuns.summarizeStatus,
|
|
227761
|
+
finalizeStatus: documentPipelineRuns.finalizeStatus,
|
|
227762
|
+
totalBatches: documentPipelineRuns.totalBatches,
|
|
227763
|
+
completedBatches: documentPipelineRuns.completedBatches,
|
|
227764
|
+
failedBatches: documentPipelineRuns.failedBatches,
|
|
227765
|
+
updatedAt: documentPipelineRuns.updatedAt
|
|
227766
|
+
}).from(documentPipelineRuns).where(eq(documentPipelineRuns.documentId, document2.id)).orderBy(desc(documentPipelineRuns.updatedAt)).limit(1);
|
|
227767
|
+
return {
|
|
227768
|
+
document: document2,
|
|
227769
|
+
searchable: active.length > 0,
|
|
227770
|
+
run: run5
|
|
227771
|
+
};
|
|
227772
|
+
});
|
|
227773
|
+
if (!result) {
|
|
227774
|
+
set2.status = 404;
|
|
227775
|
+
return { error: "Document not found" };
|
|
227776
|
+
}
|
|
227777
|
+
const run4 = result.run;
|
|
227778
|
+
return {
|
|
227779
|
+
documentId: result.document.id,
|
|
227780
|
+
embeddingStatus: result.document.embeddingStatus,
|
|
227781
|
+
activeGenerationId: result.document.activeGenerationId,
|
|
227782
|
+
pendingGenerationId: result.document.pendingGenerationId,
|
|
227783
|
+
embeddingProfile: result.document.embeddingProfile,
|
|
227784
|
+
embeddingErrorCode: result.document.embeddingErrorCode,
|
|
227785
|
+
embeddingUpdatedAt: result.document.embeddingUpdatedAt,
|
|
227786
|
+
searchable: result.searchable,
|
|
227787
|
+
pipeline: run4 ? {
|
|
227788
|
+
documentId: run4.documentId,
|
|
227789
|
+
generationId: run4.generationId,
|
|
227790
|
+
status: run4.status,
|
|
227791
|
+
revision: run4.revision,
|
|
227792
|
+
stages: {
|
|
227793
|
+
prepare: run4.prepareStatus,
|
|
227794
|
+
embed: run4.embedStatus,
|
|
227795
|
+
graph: run4.graphStatus,
|
|
227796
|
+
summarize: run4.summarizeStatus,
|
|
227797
|
+
finalize: run4.finalizeStatus
|
|
227798
|
+
},
|
|
227799
|
+
batches: {
|
|
227800
|
+
total: run4.totalBatches,
|
|
227801
|
+
completed: run4.completedBatches,
|
|
227802
|
+
failed: run4.failedBatches
|
|
227803
|
+
},
|
|
227804
|
+
updatedAt: run4.updatedAt
|
|
227805
|
+
} : null
|
|
227806
|
+
};
|
|
227807
|
+
}).post("/documents/:id/index/refresh", async ({ params: params2, set: set2, request }) => {
|
|
227808
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
227809
|
+
if (!parsedParams.success) {
|
|
227810
|
+
set2.status = 400;
|
|
227811
|
+
return { error: "Invalid document id" };
|
|
227812
|
+
}
|
|
227813
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227814
|
+
const rl = await writeRateLimiter(ip, request);
|
|
227815
|
+
if (!rl.allowed) {
|
|
227816
|
+
set2.status = 429;
|
|
227817
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
227818
|
+
return { error: "Too many requests" };
|
|
227819
|
+
}
|
|
227820
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
227821
|
+
const access = await resolveContentAccess(request);
|
|
227822
|
+
const ctx = access.ctx;
|
|
227823
|
+
if (ctx.role === "none") {
|
|
227824
|
+
set2.status = 401;
|
|
227825
|
+
return { error: "Unauthorized" };
|
|
227826
|
+
}
|
|
227827
|
+
if (!canAccessContent(access, "edit")) {
|
|
227828
|
+
set2.status = 403;
|
|
227829
|
+
return { error: "Forbidden" };
|
|
227830
|
+
}
|
|
227831
|
+
const [document2] = await withTenant(ctx, (tx) => tx.select({
|
|
227832
|
+
id: documents.id,
|
|
227833
|
+
ownerId: documents.ownerId,
|
|
227834
|
+
title: documents.title,
|
|
227835
|
+
content: documents.content
|
|
227836
|
+
}).from(documents).where(and(eq(documents.id, parsedParams.data.id), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1));
|
|
227837
|
+
if (!document2) {
|
|
227838
|
+
set2.status = 404;
|
|
227839
|
+
return { error: "Document not found" };
|
|
227840
|
+
}
|
|
227841
|
+
const queued = await enqueueDocumentPipeline({
|
|
227842
|
+
documentId: document2.id,
|
|
227843
|
+
ownerId: document2.ownerId,
|
|
227844
|
+
workspaceId: ctx.workspaceId,
|
|
227845
|
+
revision: contentHash(document2.title, document2.content ?? ""),
|
|
227846
|
+
source: "api"
|
|
227847
|
+
});
|
|
227848
|
+
set2.status = 202;
|
|
227849
|
+
return { documentId: document2.id, ...queued };
|
|
227611
227850
|
}).get("/documents/:id", async ({ params: params2, set: set2, request }) => {
|
|
227612
227851
|
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227613
227852
|
const rl = await documentRateLimiter(ip, request);
|
|
@@ -229040,14 +229279,15 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
|
|
|
229040
229279
|
set2.status = 400;
|
|
229041
229280
|
return { error: "Invalid query", details: parsed.error.flatten() };
|
|
229042
229281
|
}
|
|
229043
|
-
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229044
|
-
return { entities: [] };
|
|
229045
|
-
}
|
|
229046
229282
|
try {
|
|
229047
229283
|
const allowedIds = await allowedGraphDocumentIds(access, [
|
|
229048
229284
|
parsed.data.docId
|
|
229049
229285
|
]);
|
|
229050
|
-
if (!allowedIds.has(parsed.data.docId))
|
|
229286
|
+
if (!allowedIds.has(parsed.data.docId)) {
|
|
229287
|
+
set2.status = 404;
|
|
229288
|
+
return { error: "Document not found" };
|
|
229289
|
+
}
|
|
229290
|
+
if (!config3.GRAPH_SEARCH_ENABLED)
|
|
229051
229291
|
return { entities: [] };
|
|
229052
229292
|
const entities = await fetchDocumentEntities(parsed.data.docId);
|
|
229053
229293
|
return { entities };
|
|
@@ -229079,14 +229319,15 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
|
|
|
229079
229319
|
set2.status = 400;
|
|
229080
229320
|
return { error: "Invalid params", details: parsed.error.flatten() };
|
|
229081
229321
|
}
|
|
229082
|
-
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229083
|
-
return { related: [] };
|
|
229084
|
-
}
|
|
229085
229322
|
try {
|
|
229086
229323
|
const seedIds = await allowedGraphDocumentIds(access, [
|
|
229087
229324
|
parsed.data.docId
|
|
229088
229325
|
]);
|
|
229089
|
-
if (!seedIds.has(parsed.data.docId))
|
|
229326
|
+
if (!seedIds.has(parsed.data.docId)) {
|
|
229327
|
+
set2.status = 404;
|
|
229328
|
+
return { error: "Document not found" };
|
|
229329
|
+
}
|
|
229330
|
+
if (!config3.GRAPH_SEARCH_ENABLED)
|
|
229090
229331
|
return { related: [] };
|
|
229091
229332
|
const related = await fetchRelatedDocuments(ctx, parsed.data.docId, access);
|
|
229092
229333
|
return { related };
|
|
@@ -229119,12 +229360,16 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
|
|
|
229119
229360
|
return { error: "Invalid body", details: parsed.error.flatten() };
|
|
229120
229361
|
}
|
|
229121
229362
|
const { query, docIds, maxResults } = parsed.data;
|
|
229122
|
-
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229123
|
-
return { query, entities: [], relatedDocs: [] };
|
|
229124
|
-
}
|
|
229125
229363
|
try {
|
|
229126
229364
|
const authorizedSeeds = await allowedGraphDocumentIds(access, docIds);
|
|
229127
|
-
|
|
229365
|
+
if (docIds.some((id3) => !authorizedSeeds.has(id3))) {
|
|
229366
|
+
set2.status = 404;
|
|
229367
|
+
return { error: "One or more documents were not found" };
|
|
229368
|
+
}
|
|
229369
|
+
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229370
|
+
return { query, entities: [], relatedDocs: [] };
|
|
229371
|
+
}
|
|
229372
|
+
const result = await graphRagLookup(ctx, docIds, maxResults, access);
|
|
229128
229373
|
return { query, ...result };
|
|
229129
229374
|
} catch (err) {
|
|
229130
229375
|
logger3.warn({ err, docIds: docIds.length }, "Graph RAG search failed \u2014 returning empty");
|
|
@@ -230196,7 +230441,6 @@ async function retrieveVector(ctx, plan, limit, minimum, chunkLimit, execute, pr
|
|
|
230196
230441
|
JOIN documents d ON d.id = de.document_id
|
|
230197
230442
|
WHERE d.owner_id = ${ctx.userId}
|
|
230198
230443
|
${scope}
|
|
230199
|
-
AND d.embedding_status = 'ready'
|
|
230200
230444
|
AND d.active_embedding_generation IS NOT NULL
|
|
230201
230445
|
AND de.generation_id = d.active_embedding_generation
|
|
230202
230446
|
AND de.is_valid = true
|
|
@@ -230928,7 +231172,6 @@ async function hydrateResults(ctx, items, includeChunks, query, allowedDocumentI
|
|
|
230928
231172
|
JOIN documents d ON d.id = de.document_id
|
|
230929
231173
|
WHERE de.document_id IN (${sql.join(ids.map((id3) => sql`${id3}`), sql`, `)})
|
|
230930
231174
|
AND ${tenantOwnerSql("d", ctx)}
|
|
230931
|
-
AND d.embedding_status = 'ready'
|
|
230932
231175
|
AND d.active_embedding_generation IS NOT NULL
|
|
230933
231176
|
AND de.generation_id = d.active_embedding_generation
|
|
230934
231177
|
AND de.is_valid = true
|
|
@@ -232696,6 +232939,116 @@ async function ensureApiKeyOwner() {
|
|
|
232696
232939
|
init_config();
|
|
232697
232940
|
init_logger3();
|
|
232698
232941
|
|
|
232942
|
+
// ../../backend/src/lib/reembed-cron.ts
|
|
232943
|
+
init_schema2();
|
|
232944
|
+
init_drizzle_orm();
|
|
232945
|
+
init_config();
|
|
232946
|
+
|
|
232947
|
+
// ../../backend/src/lib/cron-timer-registry.ts
|
|
232948
|
+
function createCronTimerRegistry(clear = clearInterval) {
|
|
232949
|
+
const handles = [];
|
|
232950
|
+
let closed = false;
|
|
232951
|
+
return {
|
|
232952
|
+
register(handle) {
|
|
232953
|
+
if (closed) {
|
|
232954
|
+
clear(handle);
|
|
232955
|
+
return;
|
|
232956
|
+
}
|
|
232957
|
+
handles.push(handle);
|
|
232958
|
+
},
|
|
232959
|
+
close() {
|
|
232960
|
+
if (closed)
|
|
232961
|
+
return;
|
|
232962
|
+
closed = true;
|
|
232963
|
+
for (const handle of handles)
|
|
232964
|
+
clear(handle);
|
|
232965
|
+
handles.length = 0;
|
|
232966
|
+
}
|
|
232967
|
+
};
|
|
232968
|
+
}
|
|
232969
|
+
|
|
232970
|
+
// ../../backend/src/lib/reembed-cron.ts
|
|
232971
|
+
init_logger3();
|
|
232972
|
+
var METADATA_DEBOUNCE_MINUTES = 3;
|
|
232973
|
+
var CRON_BATCH_SIZE = 100;
|
|
232974
|
+
var CRON_TENANT = adminTenantContext(ZERO_UUID);
|
|
232975
|
+
function startReembedCron() {
|
|
232976
|
+
const timers = createCronTimerRegistry();
|
|
232977
|
+
const metadataIntervalMs = config3.METADATA_REEMBED_CRON_INTERVAL_MINUTES * 60 * 1000;
|
|
232978
|
+
if (metadataIntervalMs > 0) {
|
|
232979
|
+
logger3.info({ intervalMinutes: config3.METADATA_REEMBED_CRON_INTERVAL_MINUTES }, "Reembed metadata-stale cron started");
|
|
232980
|
+
processStaleMetadataChanges().catch((err) => {
|
|
232981
|
+
logger3.error({ err }, "Reembed metadata-stale cron initial run failed");
|
|
232982
|
+
});
|
|
232983
|
+
timers.register(setInterval(() => {
|
|
232984
|
+
processStaleMetadataChanges().catch((err) => {
|
|
232985
|
+
logger3.error({ err }, "Reembed metadata-stale cron tick failed");
|
|
232986
|
+
});
|
|
232987
|
+
}, metadataIntervalMs));
|
|
232988
|
+
} else {
|
|
232989
|
+
logger3.warn("METADATA_REEMBED_CRON_INTERVAL_MINUTES=0 \u2014 metadata-stale cron is DISABLED. " + "Metadata-only changes will NOT trigger re-embed until the cron is re-enabled.");
|
|
232990
|
+
}
|
|
232991
|
+
const idleIntervalMs = config3.REEMBED_CRON_INTERVAL_MINUTES * 60 * 1000;
|
|
232992
|
+
if (idleIntervalMs > 0) {
|
|
232993
|
+
logger3.info({ intervalMinutes: config3.REEMBED_CRON_INTERVAL_MINUTES }, "Reembed idle-pending cron started");
|
|
232994
|
+
processIdlePendingChanges().catch((err) => {
|
|
232995
|
+
logger3.error({ err }, "Reembed idle-pending cron initial run failed");
|
|
232996
|
+
});
|
|
232997
|
+
timers.register(setInterval(() => {
|
|
232998
|
+
processIdlePendingChanges().catch((err) => {
|
|
232999
|
+
logger3.error({ err }, "Reembed idle-pending cron tick failed");
|
|
233000
|
+
});
|
|
233001
|
+
}, idleIntervalMs));
|
|
233002
|
+
} else {
|
|
233003
|
+
logger3.warn("REEMBED_CRON_INTERVAL_MINUTES=0 \u2014 idle-pending cron is DISABLED. " + "Sub-threshold edits will NOT trigger catch-up embeds until the cron is re-enabled.");
|
|
233004
|
+
}
|
|
233005
|
+
return { close: () => timers.close() };
|
|
233006
|
+
}
|
|
233007
|
+
async function processStaleMetadataChanges() {
|
|
233008
|
+
const cutoff = new Date(Date.now() - METADATA_DEBOUNCE_MINUTES * 60 * 1000);
|
|
233009
|
+
const stale = await withTenant(CRON_TENANT, (tx) => tx.select({
|
|
233010
|
+
id: documents.id,
|
|
233011
|
+
workspaceId: documents.workspaceId,
|
|
233012
|
+
metadataChangedAt: documents.metadataChangedAt
|
|
233013
|
+
}).from(documents).where(and(isNotNull(documents.metadataChangedAt), lt(documents.metadataChangedAt, cutoff))).orderBy(documents.metadataChangedAt).limit(CRON_BATCH_SIZE));
|
|
233014
|
+
if (stale.length === 0)
|
|
233015
|
+
return;
|
|
233016
|
+
logger3.debug({ count: stale.length }, "Reembed cron: processing stale metadata changes");
|
|
233017
|
+
let enqueued = 0;
|
|
233018
|
+
for (const row of stale) {
|
|
233019
|
+
const originalTimestamp = row.metadataChangedAt instanceof Date ? row.metadataChangedAt.toISOString() : String(row.metadataChangedAt);
|
|
233020
|
+
const cleared = await withTenant(CRON_TENANT, (tx) => tx.execute(sql`
|
|
233021
|
+
UPDATE documents
|
|
233022
|
+
SET metadata_changed_at = NULL
|
|
233023
|
+
WHERE id = ${row.id}
|
|
233024
|
+
AND metadata_changed_at = ${originalTimestamp}
|
|
233025
|
+
RETURNING id
|
|
233026
|
+
`));
|
|
233027
|
+
if (Array.isArray(cleared) && cleared.length > 0) {
|
|
233028
|
+
await enqueueReembed([row.id], row.workspaceId ?? undefined);
|
|
233029
|
+
enqueued += 1;
|
|
233030
|
+
}
|
|
233031
|
+
}
|
|
233032
|
+
if (enqueued > 0) {
|
|
233033
|
+
logger3.info({ scanned: stale.length, enqueued }, "Reembed cron: enqueued stale metadata changes");
|
|
233034
|
+
}
|
|
233035
|
+
}
|
|
233036
|
+
async function processIdlePendingChanges() {
|
|
233037
|
+
const cutoff = new Date(Date.now() - config3.REEMBED_MAX_IDLE_HOURS * 3600000);
|
|
233038
|
+
const idle = await withTenant(CRON_TENANT, (tx) => tx.select({ id: documents.id, workspaceId: documents.workspaceId }).from(documents).where(and(eq(documents.pendingMinorChanges, true), lt(documents.lastSignificantUpdateAt, cutoff))).orderBy(documents.lastSignificantUpdateAt).limit(CRON_BATCH_SIZE));
|
|
233039
|
+
if (idle.length === 0)
|
|
233040
|
+
return;
|
|
233041
|
+
logger3.debug({ count: idle.length }, "Reembed cron: processing idle pending-minor changes");
|
|
233042
|
+
let enqueued = 0;
|
|
233043
|
+
for (const row of idle) {
|
|
233044
|
+
await enqueueReembed([row.id], row.workspaceId ?? undefined);
|
|
233045
|
+
enqueued += 1;
|
|
233046
|
+
}
|
|
233047
|
+
if (enqueued > 0) {
|
|
233048
|
+
logger3.info({ scanned: idle.length, enqueued }, "Reembed cron: enqueued idle pending-minor changes");
|
|
233049
|
+
}
|
|
233050
|
+
}
|
|
233051
|
+
|
|
232699
233052
|
// ../../backend/src/queue/adapters.ts
|
|
232700
233053
|
init_schema2();
|
|
232701
233054
|
init_drizzle_orm();
|
|
@@ -232749,15 +233102,62 @@ async function activateEmbeddingGeneration(documentId, generationId, expectedChu
|
|
|
232749
233102
|
// ../../backend/src/queue/adapters.ts
|
|
232750
233103
|
init_config();
|
|
232751
233104
|
|
|
233105
|
+
// ../../backend/src/lib/graph/generation-state.ts
|
|
233106
|
+
function literal2(value) {
|
|
233107
|
+
return JSON.stringify(value);
|
|
233108
|
+
}
|
|
233109
|
+
function isStaleRevisionError(error53) {
|
|
233110
|
+
return error53 instanceof Error && error53.name === "stale_revision";
|
|
233111
|
+
}
|
|
233112
|
+
async function runGenerationFencedGraphWrite(dependencies) {
|
|
233113
|
+
if (!await dependencies.lockCurrentGeneration()) {
|
|
233114
|
+
const error53 = new Error("graph generation is stale");
|
|
233115
|
+
error53.name = "stale_revision";
|
|
233116
|
+
throw error53;
|
|
233117
|
+
}
|
|
233118
|
+
await dependencies.persist();
|
|
233119
|
+
}
|
|
233120
|
+
function graphReplacementCyphers(identity) {
|
|
233121
|
+
const documentId = literal2(identity.documentId);
|
|
233122
|
+
const generationId = literal2(identity.generationId);
|
|
233123
|
+
const revision = literal2(identity.revision);
|
|
233124
|
+
const timestamp3 = literal2(identity.timestamp);
|
|
233125
|
+
return [
|
|
233126
|
+
`MATCH (d:Document {id: ${documentId}})-[legacy:MENTIONS]->()
|
|
233127
|
+
DELETE legacy RETURN count(legacy)`,
|
|
233128
|
+
`MATCH ()-[r]->() WHERE r.document_id = ${documentId}
|
|
233129
|
+
DELETE r RETURN count(r)`,
|
|
233130
|
+
`MERGE (d:Document {id: ${documentId}})
|
|
233131
|
+
SET d.generation_id = ${generationId}, d.revision = ${revision},
|
|
233132
|
+
d.created_at = coalesce(d.created_at, ${timestamp3}),
|
|
233133
|
+
d.entity_extracted_at = ${timestamp3}
|
|
233134
|
+
RETURN d.id`
|
|
233135
|
+
];
|
|
233136
|
+
}
|
|
233137
|
+
function graphCompensationCyphers(documentIdValue, generationIdValue) {
|
|
233138
|
+
const documentId = literal2(documentIdValue);
|
|
233139
|
+
const generationId = literal2(generationIdValue);
|
|
233140
|
+
return [
|
|
233141
|
+
`MATCH ()-[r]->()
|
|
233142
|
+
WHERE r.document_id = ${documentId}
|
|
233143
|
+
AND r.generation_id = ${generationId}
|
|
233144
|
+
DELETE r RETURN count(r)`,
|
|
233145
|
+
`MATCH (d:Document {id: ${documentId}})
|
|
233146
|
+
WHERE d.generation_id = ${generationId}
|
|
233147
|
+
DETACH DELETE d RETURN count(d)`
|
|
233148
|
+
];
|
|
233149
|
+
}
|
|
233150
|
+
|
|
232752
233151
|
// ../../backend/src/lib/graph/delete-document-state.ts
|
|
232753
|
-
async function
|
|
233152
|
+
async function deleteDocumentGraphGeneration(documentId, generationId) {
|
|
232754
233153
|
const sql6 = await getGraphDb();
|
|
232755
233154
|
if (!sql6)
|
|
232756
233155
|
return;
|
|
232757
|
-
const literal2 = JSON.stringify(documentId);
|
|
232758
233156
|
await sql6.begin(async (tx) => {
|
|
232759
233157
|
await tx.unsafe("SET LOCAL search_path = ag_catalog, public");
|
|
232760
|
-
|
|
233158
|
+
for (const cypher of graphCompensationCyphers(documentId, generationId)) {
|
|
233159
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (deleted agtype)`);
|
|
233160
|
+
}
|
|
232761
233161
|
});
|
|
232762
233162
|
}
|
|
232763
233163
|
|
|
@@ -232780,8 +233180,9 @@ var RELATION_TYPES = [
|
|
|
232780
233180
|
"AUTHORED_BY"
|
|
232781
233181
|
];
|
|
232782
233182
|
var EXTRACT_DEDUP_TTL_SECONDS = 24 * 60 * 60;
|
|
232783
|
-
function extractDedupKey(documentId, chunkIndex, chunkHash2) {
|
|
232784
|
-
|
|
233183
|
+
function extractDedupKey(documentId, chunkIndex, chunkHash2, generationId) {
|
|
233184
|
+
const generation = generationId ? `${generationId}:` : "";
|
|
233185
|
+
return `hiai-docs:extract:done:${documentId}:${generation}${chunkIndex}:${chunkHash2}`;
|
|
232785
233186
|
}
|
|
232786
233187
|
var DEFAULT_MAX_TOKENS = 1024;
|
|
232787
233188
|
var DEFAULT_TEMPERATURE = 0;
|
|
@@ -232802,12 +233203,24 @@ function setCachedEntity(name, type, confidence) {
|
|
|
232802
233203
|
async function extractEntities(chunkText2, documentId, options = {}) {
|
|
232803
233204
|
if (!config3.GRAPH_EXTRACT_ENABLED)
|
|
232804
233205
|
return [];
|
|
233206
|
+
let claimedDedupKey;
|
|
233207
|
+
const releaseDedupClaim = async () => {
|
|
233208
|
+
if (!claimedDedupKey)
|
|
233209
|
+
return;
|
|
233210
|
+
try {
|
|
233211
|
+
await redis.del(claimedDedupKey);
|
|
233212
|
+
} catch (err) {
|
|
233213
|
+
logger3.warn({ err, documentId, claimedDedupKey }, "Failed to release extract-dedup slot");
|
|
233214
|
+
}
|
|
233215
|
+
};
|
|
232805
233216
|
if (options.chunkHash && options.chunkIndex !== undefined) {
|
|
232806
|
-
const key = extractDedupKey(documentId, options.chunkIndex, options.chunkHash);
|
|
233217
|
+
const key = extractDedupKey(documentId, options.chunkIndex, options.chunkHash, options.generationId);
|
|
232807
233218
|
let acquired = false;
|
|
232808
233219
|
try {
|
|
232809
233220
|
const result = await redis.set(key, "1", "EX", EXTRACT_DEDUP_TTL_SECONDS, "NX");
|
|
232810
233221
|
acquired = result === "OK";
|
|
233222
|
+
if (acquired)
|
|
233223
|
+
claimedDedupKey = key;
|
|
232811
233224
|
} catch (err) {
|
|
232812
233225
|
logger3.warn({ err, documentId, chunkIndex: options.chunkIndex }, "Extract-dedup Redis SET failed \u2014 falling through to extraction");
|
|
232813
233226
|
acquired = true;
|
|
@@ -232816,22 +233229,31 @@ async function extractEntities(chunkText2, documentId, options = {}) {
|
|
|
232816
233229
|
return [];
|
|
232817
233230
|
}
|
|
232818
233231
|
const sql6 = await getGraphDb();
|
|
232819
|
-
if (!sql6)
|
|
233232
|
+
if (!sql6) {
|
|
233233
|
+
await releaseDedupClaim();
|
|
232820
233234
|
return [];
|
|
232821
|
-
|
|
233235
|
+
}
|
|
233236
|
+
if (!chunkText2) {
|
|
233237
|
+
await releaseDedupClaim();
|
|
232822
233238
|
return [];
|
|
233239
|
+
}
|
|
232823
233240
|
let entities;
|
|
232824
233241
|
try {
|
|
232825
233242
|
entities = await callEntityExtractionLLM(chunkText2, options);
|
|
232826
233243
|
} catch (err) {
|
|
233244
|
+
await releaseDedupClaim();
|
|
232827
233245
|
logger3.warn({ err, documentId }, "Entity extraction LLM call failed \u2014 skipping");
|
|
232828
233246
|
return [];
|
|
232829
233247
|
}
|
|
232830
|
-
if (entities.length === 0)
|
|
232831
|
-
return [];
|
|
232832
233248
|
try {
|
|
232833
|
-
await persistEntities(sql6, documentId, entities
|
|
233249
|
+
await persistEntities(sql6, documentId, entities, {
|
|
233250
|
+
generationId: options.generationId ?? "legacy",
|
|
233251
|
+
revision: options.revision ?? "legacy"
|
|
233252
|
+
});
|
|
232834
233253
|
} catch (err) {
|
|
233254
|
+
await releaseDedupClaim();
|
|
233255
|
+
if (isStaleRevisionError(err))
|
|
233256
|
+
throw err;
|
|
232835
233257
|
logger3.warn({ err, documentId, count: entities.length }, "Failed to persist extracted entities to AGE \u2014 discarding");
|
|
232836
233258
|
return [];
|
|
232837
233259
|
}
|
|
@@ -233000,30 +233422,55 @@ function stripMarkdownFences(text4) {
|
|
|
233000
233422
|
const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(text4.trim());
|
|
233001
233423
|
return fenced?.[1] ?? text4;
|
|
233002
233424
|
}
|
|
233003
|
-
async function persistEntities(sql6, documentId, entities) {
|
|
233425
|
+
async function persistEntities(sql6, documentId, entities, identity) {
|
|
233004
233426
|
await sql6.begin(async (tx) => {
|
|
233005
233427
|
await tx`SELECT pg_catalog.set_config('search_path', 'ag_catalog, "$user", public', false)`;
|
|
233428
|
+
await tx`SELECT pg_catalog.set_config('app.current_user_id', '00000000-0000-0000-0000-000000000000', true)`;
|
|
233429
|
+
await tx`SELECT pg_catalog.set_config('app.current_user_role', 'admin', true)`;
|
|
233430
|
+
await tx`SELECT pg_catalog.set_config('app.current_workspace_id', '', true)`;
|
|
233006
233431
|
const nowIso = new Date().toISOString();
|
|
233007
|
-
|
|
233008
|
-
|
|
233009
|
-
|
|
233010
|
-
|
|
233011
|
-
|
|
233012
|
-
|
|
233013
|
-
|
|
233014
|
-
|
|
233015
|
-
|
|
233016
|
-
|
|
233017
|
-
|
|
233018
|
-
|
|
233019
|
-
|
|
233020
|
-
|
|
233021
|
-
|
|
233022
|
-
|
|
233023
|
-
|
|
233024
|
-
|
|
233432
|
+
await runGenerationFencedGraphWrite({
|
|
233433
|
+
async lockCurrentGeneration() {
|
|
233434
|
+
const rows = await tx`
|
|
233435
|
+
SELECT d.id
|
|
233436
|
+
FROM public.documents d
|
|
233437
|
+
JOIN public.document_pipeline_runs r
|
|
233438
|
+
ON r.document_id = d.id
|
|
233439
|
+
AND r.generation_id = ${identity.generationId}::uuid
|
|
233440
|
+
WHERE d.id = ${documentId}::uuid
|
|
233441
|
+
AND d.active_embedding_generation = ${identity.generationId}::uuid
|
|
233442
|
+
AND d.content_hash = ${identity.revision}
|
|
233443
|
+
AND r.revision = ${identity.revision}
|
|
233444
|
+
AND r.status NOT IN ('cancelled', 'failed')
|
|
233445
|
+
AND r.graph_status = 'processing'
|
|
233446
|
+
FOR UPDATE OF d, r
|
|
233447
|
+
`;
|
|
233448
|
+
return rows.length === 1;
|
|
233449
|
+
},
|
|
233450
|
+
async persist() {
|
|
233451
|
+
for (const cypher of graphReplacementCyphers({
|
|
233452
|
+
documentId,
|
|
233453
|
+
generationId: identity.generationId,
|
|
233454
|
+
revision: identity.revision,
|
|
233455
|
+
timestamp: nowIso
|
|
233456
|
+
})) {
|
|
233457
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
|
|
233458
|
+
}
|
|
233459
|
+
for (const ent of entities) {
|
|
233460
|
+
const label = ent.type;
|
|
233461
|
+
const name = ent.name;
|
|
233462
|
+
const conf = ent.confidence;
|
|
233463
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${entityUpsertCypher(documentId, label, name, nowIso, conf)} $$) AS (result agtype)`);
|
|
233464
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${documentEntityEdgeCypher(documentId, identity.generationId, label, name, conf)} $$) AS (result agtype)`);
|
|
233465
|
+
}
|
|
233466
|
+
for (const ent of entities) {
|
|
233467
|
+
for (const rel of ent.relationships) {
|
|
233468
|
+
const cypher = entityRelationCypher(documentId, identity.generationId, ent.type, ent.name, rel.targetName, rel.relationType, rel.confidence);
|
|
233469
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
|
|
233470
|
+
}
|
|
233471
|
+
}
|
|
233025
233472
|
}
|
|
233026
|
-
}
|
|
233473
|
+
});
|
|
233027
233474
|
});
|
|
233028
233475
|
}
|
|
233029
233476
|
function entityUpsertCypher(docId, label, name, nowIso, confidence) {
|
|
@@ -233032,27 +233479,118 @@ function entityUpsertCypher(docId, label, name, nowIso, confidence) {
|
|
|
233032
233479
|
MERGE (e:\`${label}\` {name: $name})
|
|
233033
233480
|
SET e.created_at = $now, e.last_seen_doc = $docId, e.confidence = $conf
|
|
233034
233481
|
RETURN e.name
|
|
233035
|
-
`.
|
|
233482
|
+
`.replaceAll("$name", JSON.stringify(name)).replaceAll("$now", JSON.stringify(nowIso)).replaceAll("$docId", JSON.stringify(docId)).replaceAll("$conf", confLiteral);
|
|
233036
233483
|
}
|
|
233037
|
-
function documentEntityEdgeCypher(docId, label, name, confidence) {
|
|
233484
|
+
function documentEntityEdgeCypher(docId, generationId, label, name, confidence) {
|
|
233038
233485
|
const confLiteral = typeof confidence === "number" ? JSON.stringify(confidence) : "null";
|
|
233039
233486
|
return `
|
|
233040
233487
|
MATCH (d:Document {id: $docId})
|
|
233041
233488
|
MATCH (e:\`${label}\` {name: $name})
|
|
233042
|
-
MERGE (d)-[r:MENTIONS]->(e)
|
|
233043
|
-
SET r.created_at = $now, r.confidence = $conf
|
|
233489
|
+
MERGE (d)-[r:MENTIONS {document_id: $docId}]->(e)
|
|
233490
|
+
SET r.created_at = $now, r.confidence = $conf, r.generation_id = $generationId
|
|
233044
233491
|
RETURN r
|
|
233045
|
-
`.
|
|
233492
|
+
`.replaceAll("$docId", JSON.stringify(docId)).replaceAll("$name", JSON.stringify(name)).replaceAll("$now", JSON.stringify(new Date().toISOString())).replaceAll("$generationId", JSON.stringify(generationId)).replaceAll("$conf", confLiteral);
|
|
233046
233493
|
}
|
|
233047
|
-
function entityRelationCypher(sourceLabel, sourceName, targetName, relationType, confidence) {
|
|
233494
|
+
function entityRelationCypher(documentId, generationId, sourceLabel, sourceName, targetName, relationType, confidence) {
|
|
233048
233495
|
const confLiteral = typeof confidence === "number" ? JSON.stringify(confidence) : "null";
|
|
233049
233496
|
return `
|
|
233050
233497
|
MATCH (a:\`${sourceLabel}\` {name: $source})
|
|
233051
233498
|
MATCH (b {name: $target})
|
|
233052
|
-
MERGE (a)-[r:\`${relationType}\`]->(b)
|
|
233053
|
-
SET r.created_at = $now, r.confidence = $conf
|
|
233499
|
+
MERGE (a)-[r:\`${relationType}\` {document_id: $documentId}]->(b)
|
|
233500
|
+
SET r.created_at = $now, r.confidence = $conf, r.generation_id = $generationId
|
|
233054
233501
|
RETURN r
|
|
233055
|
-
`.
|
|
233502
|
+
`.replaceAll("$source", JSON.stringify(sourceName)).replaceAll("$target", JSON.stringify(targetName)).replaceAll("$documentId", JSON.stringify(documentId)).replaceAll("$generationId", JSON.stringify(generationId)).replaceAll("$now", JSON.stringify(new Date().toISOString())).replaceAll("$conf", confLiteral);
|
|
233503
|
+
}
|
|
233504
|
+
|
|
233505
|
+
// ../../backend/src/lib/knowledge-summary.ts
|
|
233506
|
+
async function runKnowledgeSummaryStage(dependencies) {
|
|
233507
|
+
const document2 = await dependencies.readCurrent();
|
|
233508
|
+
if (!document2)
|
|
233509
|
+
return "cancelled";
|
|
233510
|
+
const summary = await dependencies.generate(document2);
|
|
233511
|
+
if (summary.status === "skipped")
|
|
233512
|
+
return "skipped";
|
|
233513
|
+
return await dependencies.persistIfCurrent(summary) ? "ready" : "cancelled";
|
|
233514
|
+
}
|
|
233515
|
+
async function buildKnowledgeSummary(document2, provider) {
|
|
233516
|
+
if (!document2.content.trim()) {
|
|
233517
|
+
return { status: "skipped", reason: "empty_document" };
|
|
233518
|
+
}
|
|
233519
|
+
const output = await provider(document2);
|
|
233520
|
+
if (!output)
|
|
233521
|
+
throw new Error("summary_provider_failed");
|
|
233522
|
+
const language = output.language.trim();
|
|
233523
|
+
const description = output.description.trim();
|
|
233524
|
+
if (!language || !description)
|
|
233525
|
+
throw new Error("summary_provider_invalid");
|
|
233526
|
+
const seen = new Set;
|
|
233527
|
+
const keywords = [];
|
|
233528
|
+
for (const raw of output.keywords) {
|
|
233529
|
+
const keyword = raw.trim();
|
|
233530
|
+
const key = keyword.toLocaleLowerCase("en");
|
|
233531
|
+
if (!keyword || seen.has(key))
|
|
233532
|
+
continue;
|
|
233533
|
+
seen.add(key);
|
|
233534
|
+
keywords.push(keyword);
|
|
233535
|
+
if (keywords.length === 20)
|
|
233536
|
+
break;
|
|
233537
|
+
}
|
|
233538
|
+
return {
|
|
233539
|
+
status: "ready",
|
|
233540
|
+
language,
|
|
233541
|
+
description,
|
|
233542
|
+
keywords,
|
|
233543
|
+
model: output.model
|
|
233544
|
+
};
|
|
233545
|
+
}
|
|
233546
|
+
|
|
233547
|
+
// ../../backend/src/lib/knowledge-summary-provider.ts
|
|
233548
|
+
init_zod();
|
|
233549
|
+
init_config();
|
|
233550
|
+
var summarySchema = exports_external2.object({
|
|
233551
|
+
language: exports_external2.string().min(1).max(32),
|
|
233552
|
+
description: exports_external2.string().min(1).max(2000),
|
|
233553
|
+
keywords: exports_external2.array(exports_external2.string().min(1).max(100)).max(20)
|
|
233554
|
+
});
|
|
233555
|
+
var systemPrompt = [
|
|
233556
|
+
"Summarize a knowledge-base document for retrieval metadata.",
|
|
233557
|
+
"Return only JSON with language, description, and keywords.",
|
|
233558
|
+
"Use a concise factual description and at most 20 distinct keywords."
|
|
233559
|
+
].join(`
|
|
233560
|
+
`);
|
|
233561
|
+
function provider(baseUrl, apiKey, model) {
|
|
233562
|
+
if (!baseUrl || !model)
|
|
233563
|
+
return;
|
|
233564
|
+
return {
|
|
233565
|
+
baseUrl,
|
|
233566
|
+
model,
|
|
233567
|
+
apiKey: resolveChatProviderKey(baseUrl, apiKey, config3.OPENROUTER_API_KEY),
|
|
233568
|
+
timeoutMs: config3.GRAPH_EXTRACT_TIMEOUT_MS,
|
|
233569
|
+
reasoningEffort: config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
233570
|
+
};
|
|
233571
|
+
}
|
|
233572
|
+
async function requestKnowledgeSummary(document2) {
|
|
233573
|
+
const primary = provider(config3.GRAPH_EXTRACT_BASE_URL, config3.GRAPH_EXTRACT_API_KEY, config3.GRAPH_EXTRACT_MODEL);
|
|
233574
|
+
if (!primary)
|
|
233575
|
+
return null;
|
|
233576
|
+
const fallback = provider(config3.GRAPH_EXTRACT_FALLBACK_BASE_URL, config3.GRAPH_EXTRACT_FALLBACK_API_KEY, config3.GRAPH_EXTRACT_FALLBACK_MODEL);
|
|
233577
|
+
const result = await requestStructuredChat({
|
|
233578
|
+
primary,
|
|
233579
|
+
fallback,
|
|
233580
|
+
messages: [
|
|
233581
|
+
{ role: "system", content: systemPrompt },
|
|
233582
|
+
{
|
|
233583
|
+
role: "user",
|
|
233584
|
+
content: `Title: ${document2.title}
|
|
233585
|
+
|
|
233586
|
+
${document2.content.slice(0, 50000)}`
|
|
233587
|
+
}
|
|
233588
|
+
],
|
|
233589
|
+
outputSchema: summarySchema,
|
|
233590
|
+
maxTokens: 768,
|
|
233591
|
+
temperature: 0
|
|
233592
|
+
});
|
|
233593
|
+
return result ? { ...result.data, model: result.model } : null;
|
|
233056
233594
|
}
|
|
233057
233595
|
|
|
233058
233596
|
// ../../backend/src/queue/document-revision.ts
|
|
@@ -233230,6 +233768,23 @@ async function getRun(generationId) {
|
|
|
233230
233768
|
return run4 ?? null;
|
|
233231
233769
|
});
|
|
233232
233770
|
}
|
|
233771
|
+
async function isCurrentPipelineGeneration(job2) {
|
|
233772
|
+
return withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
|
|
233773
|
+
const [current] = await tx.select({ id: documents.id }).from(documents).innerJoin(documentPipelineRuns, and(eq(documentPipelineRuns.documentId, documents.id), eq(documentPipelineRuns.generationId, job2.generationId))).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documentPipelineRuns.revision, job2.revision), ne(documentPipelineRuns.status, "cancelled"), ne(documentPipelineRuns.status, "failed"), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
233774
|
+
return Boolean(current);
|
|
233775
|
+
});
|
|
233776
|
+
}
|
|
233777
|
+
async function cancelStalePipelineRun(job2) {
|
|
233778
|
+
await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
|
|
233779
|
+
status: "cancelled",
|
|
233780
|
+
graphStatus: "cancelled",
|
|
233781
|
+
summarizeStatus: "cancelled",
|
|
233782
|
+
finalizeStatus: "cancelled",
|
|
233783
|
+
errorCode: "stale_revision",
|
|
233784
|
+
completedAt: new Date,
|
|
233785
|
+
updatedAt: new Date
|
|
233786
|
+
}).where(and(eq(documentPipelineRuns.generationId, job2.generationId), ne(documentPipelineRuns.status, "cancelled"))));
|
|
233787
|
+
}
|
|
233233
233788
|
async function setStageStatus(generationId, stage, status2, errorCode) {
|
|
233234
233789
|
await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
|
|
233235
233790
|
...stagePatch(stage, status2),
|
|
@@ -233485,18 +234040,22 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
233485
234040
|
},
|
|
233486
234041
|
async extract(job2) {
|
|
233487
234042
|
const doc5 = await withTenant(admin, async (tx) => {
|
|
233488
|
-
const [row] = await tx.select({ content: documents.content }).from(documents).where(and(eq(documents.id, job2.documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
234043
|
+
const [row] = await tx.select({ content: documents.content }).from(documents).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documents.contentHash, job2.revision), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
233489
234044
|
return row;
|
|
233490
234045
|
});
|
|
233491
234046
|
if (!doc5)
|
|
233492
234047
|
throw new Error("Pipeline document not found");
|
|
233493
|
-
await withProviderPermit(providerProfile(`graph:${config3.GRAPH_EXTRACT_MODEL ?? "default"}`), "graph", () => extractEntities(doc5.content ?? "", job2.documentId
|
|
234048
|
+
await withProviderPermit(providerProfile(`graph:${config3.GRAPH_EXTRACT_MODEL ?? "default"}`), "graph", () => extractEntities(doc5.content ?? "", job2.documentId, {
|
|
234049
|
+
generationId: job2.generationId,
|
|
234050
|
+
revision: job2.revision
|
|
234051
|
+
}));
|
|
233494
234052
|
},
|
|
233495
234053
|
async compensateExtract(job2) {
|
|
233496
234054
|
const owned = await withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => tx.select({ id: documents.id }).from(documents).where(and(eq(documents.id, job2.documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1));
|
|
233497
234055
|
if (owned.length === 1)
|
|
233498
|
-
await
|
|
234056
|
+
await deleteDocumentGraphGeneration(job2.documentId, job2.generationId);
|
|
233499
234057
|
},
|
|
234058
|
+
cancelStaleRun: cancelStalePipelineRun,
|
|
233500
234059
|
setGraphStatus: (generationId, status2, errorCode) => setStageStatus(generationId, "graph", status2, errorCode),
|
|
233501
234060
|
async enqueueSummarize(job2) {
|
|
233502
234061
|
const data = { ...job2, stage: "summarize" };
|
|
@@ -233509,6 +234068,8 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
233509
234068
|
},
|
|
233510
234069
|
summarize: {
|
|
233511
234070
|
isCancelled: async (job2) => (await getRun(job2.generationId))?.status === "cancelled",
|
|
234071
|
+
isCurrent: isCurrentPipelineGeneration,
|
|
234072
|
+
cancelStaleRun: cancelStalePipelineRun,
|
|
233512
234073
|
async getRun(job2) {
|
|
233513
234074
|
const run4 = await getRun(job2.generationId);
|
|
233514
234075
|
return run4 ? {
|
|
@@ -233519,8 +234080,59 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
233519
234080
|
embedStatus: pipelineStageStatus(run4.embedStatus)
|
|
233520
234081
|
} : null;
|
|
233521
234082
|
},
|
|
233522
|
-
enabled: () =>
|
|
233523
|
-
async summarize() {
|
|
234083
|
+
enabled: () => config3.GRAPH_EXTRACT_ENABLED && Boolean(config3.GRAPH_EXTRACT_BASE_URL && config3.GRAPH_EXTRACT_MODEL),
|
|
234084
|
+
async summarize(job2) {
|
|
234085
|
+
return runKnowledgeSummaryStage({
|
|
234086
|
+
readCurrent: () => withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
|
|
234087
|
+
const [document2] = await tx.select({
|
|
234088
|
+
title: documents.title,
|
|
234089
|
+
content: documents.content,
|
|
234090
|
+
contentHash: documents.contentHash,
|
|
234091
|
+
activeGeneration: documents.activeEmbeddingGeneration
|
|
234092
|
+
}).from(documents).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
234093
|
+
if (!document2)
|
|
234094
|
+
return null;
|
|
234095
|
+
const revision = resolveDocumentRevision(document2.contentHash, document2.title, document2.content ?? "");
|
|
234096
|
+
if (revision !== job2.revision)
|
|
234097
|
+
return null;
|
|
234098
|
+
return {
|
|
234099
|
+
title: document2.title,
|
|
234100
|
+
content: document2.content ?? "",
|
|
234101
|
+
revision
|
|
234102
|
+
};
|
|
234103
|
+
}),
|
|
234104
|
+
generate: (document2) => buildKnowledgeSummary(document2, requestKnowledgeSummary),
|
|
234105
|
+
persistIfCurrent: (summary) => withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
|
|
234106
|
+
const [current] = await tx.select({ id: documents.id }).from(documents).innerJoin(documentPipelineRuns, and(eq(documentPipelineRuns.documentId, documents.id), eq(documentPipelineRuns.generationId, job2.generationId))).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documents.contentHash, job2.revision), eq(documentPipelineRuns.revision, job2.revision), eq(documentPipelineRuns.summarizeStatus, "processing"), ne(documentPipelineRuns.status, "cancelled"), ne(documentPipelineRuns.status, "failed"), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1).for("update");
|
|
234107
|
+
if (!current)
|
|
234108
|
+
return false;
|
|
234109
|
+
await tx.insert(documentKnowledgeSummaries).values({
|
|
234110
|
+
document_id: job2.documentId,
|
|
234111
|
+
owner_id: job2.ownerId,
|
|
234112
|
+
workspace_id: job2.workspaceId,
|
|
234113
|
+
generation_id: job2.generationId,
|
|
234114
|
+
revision: job2.revision,
|
|
234115
|
+
language: summary.language,
|
|
234116
|
+
description: summary.description,
|
|
234117
|
+
keywords: summary.keywords,
|
|
234118
|
+
updated_at: new Date
|
|
234119
|
+
}).onConflictDoUpdate({
|
|
234120
|
+
target: [
|
|
234121
|
+
documentKnowledgeSummaries.document_id,
|
|
234122
|
+
documentKnowledgeSummaries.generation_id
|
|
234123
|
+
],
|
|
234124
|
+
set: {
|
|
234125
|
+
revision: job2.revision,
|
|
234126
|
+
language: summary.language,
|
|
234127
|
+
description: summary.description,
|
|
234128
|
+
keywords: summary.keywords,
|
|
234129
|
+
updated_at: new Date
|
|
234130
|
+
}
|
|
234131
|
+
});
|
|
234132
|
+
return true;
|
|
234133
|
+
})
|
|
234134
|
+
});
|
|
234135
|
+
},
|
|
233524
234136
|
setSummaryStatus: (generationId, status2, errorCode) => setStageStatus(generationId, "summarize", status2, errorCode),
|
|
233525
234137
|
async enqueueFinalize(job2) {
|
|
233526
234138
|
const data = { ...job2, stage: "finalize" };
|
|
@@ -233905,10 +234517,8 @@ function createEmbedWorker(redisUrl, deps, options = {}) {
|
|
|
233905
234517
|
});
|
|
233906
234518
|
}
|
|
233907
234519
|
|
|
233908
|
-
// ../../backend/src/queue/
|
|
234520
|
+
// ../../backend/src/queue/stage-policies.ts
|
|
233909
234521
|
function deriveFinalStatus(run4) {
|
|
233910
|
-
if (!run4)
|
|
233911
|
-
throw new Error("Pipeline run not found");
|
|
233912
234522
|
if (run4.status === "cancelled")
|
|
233913
234523
|
return "cancelled";
|
|
233914
234524
|
if (run4.embedStatus === "failed" || run4.embedStatus === "cancelled")
|
|
@@ -233923,6 +234533,96 @@ function deriveFinalStatus(run4) {
|
|
|
233923
234533
|
}
|
|
233924
234534
|
return "failed";
|
|
233925
234535
|
}
|
|
234536
|
+
async function processSummaryStage(job2, dependencies) {
|
|
234537
|
+
const isExplicitlyCancelled = async () => await dependencies.isCancelled?.(job2) === true;
|
|
234538
|
+
const hasCurrentGeneration = async () => await dependencies.isCurrent?.(job2) !== false;
|
|
234539
|
+
const continueIfCurrent = async () => {
|
|
234540
|
+
if (await isExplicitlyCancelled())
|
|
234541
|
+
return false;
|
|
234542
|
+
if (await hasCurrentGeneration())
|
|
234543
|
+
return true;
|
|
234544
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234545
|
+
return false;
|
|
234546
|
+
};
|
|
234547
|
+
if (!await continueIfCurrent())
|
|
234548
|
+
return;
|
|
234549
|
+
const run4 = await dependencies.getRun(job2);
|
|
234550
|
+
if (!run4)
|
|
234551
|
+
throw new Error("Pipeline run not found");
|
|
234552
|
+
if (run4.ownerId !== job2.ownerId || run4.documentId !== job2.documentId) {
|
|
234553
|
+
throw new Error("Pipeline owner mismatch");
|
|
234554
|
+
}
|
|
234555
|
+
if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
|
|
234556
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234557
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234558
|
+
return;
|
|
234559
|
+
}
|
|
234560
|
+
if (!dependencies.enabled()) {
|
|
234561
|
+
if (!await continueIfCurrent())
|
|
234562
|
+
return;
|
|
234563
|
+
await dependencies.setSummaryStatus(job2.generationId, "skipped");
|
|
234564
|
+
if (!await continueIfCurrent())
|
|
234565
|
+
return;
|
|
234566
|
+
await dependencies.enqueueFinalize(job2);
|
|
234567
|
+
return;
|
|
234568
|
+
}
|
|
234569
|
+
if (!await continueIfCurrent())
|
|
234570
|
+
return;
|
|
234571
|
+
await dependencies.setSummaryStatus(job2.generationId, "processing");
|
|
234572
|
+
try {
|
|
234573
|
+
if (!await continueIfCurrent())
|
|
234574
|
+
return;
|
|
234575
|
+
const status2 = await dependencies.summarize(job2);
|
|
234576
|
+
if (status2 === "cancelled") {
|
|
234577
|
+
if (await isExplicitlyCancelled()) {
|
|
234578
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled");
|
|
234579
|
+
return;
|
|
234580
|
+
}
|
|
234581
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234582
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234583
|
+
return;
|
|
234584
|
+
}
|
|
234585
|
+
if (await isExplicitlyCancelled()) {
|
|
234586
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled");
|
|
234587
|
+
return;
|
|
234588
|
+
}
|
|
234589
|
+
if (!await hasCurrentGeneration()) {
|
|
234590
|
+
if (await isExplicitlyCancelled())
|
|
234591
|
+
return;
|
|
234592
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234593
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234594
|
+
return;
|
|
234595
|
+
}
|
|
234596
|
+
await dependencies.setSummaryStatus(job2.generationId, status2);
|
|
234597
|
+
} catch (error53) {
|
|
234598
|
+
if (!await continueIfCurrent())
|
|
234599
|
+
return;
|
|
234600
|
+
await dependencies.setSummaryStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "summary_failed");
|
|
234601
|
+
if (!await continueIfCurrent())
|
|
234602
|
+
return;
|
|
234603
|
+
await dependencies.enqueueFinalize(job2);
|
|
234604
|
+
return;
|
|
234605
|
+
}
|
|
234606
|
+
if (!await continueIfCurrent())
|
|
234607
|
+
return;
|
|
234608
|
+
await dependencies.enqueueFinalize(job2);
|
|
234609
|
+
}
|
|
234610
|
+
async function processGraphStageFailure(job2, error53, dependencies) {
|
|
234611
|
+
if (isStaleRevisionError(error53)) {
|
|
234612
|
+
await dependencies.setGraphStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234613
|
+
await dependencies.cancelStaleRun(job2);
|
|
234614
|
+
return;
|
|
234615
|
+
}
|
|
234616
|
+
if (await dependencies.isCancelled?.(job2))
|
|
234617
|
+
throw error53;
|
|
234618
|
+
await dependencies.setGraphStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "graph_failed");
|
|
234619
|
+
if (!await dependencies.isCancelled?.(job2)) {
|
|
234620
|
+
await dependencies.enqueueSummarize(job2);
|
|
234621
|
+
}
|
|
234622
|
+
throw error53;
|
|
234623
|
+
}
|
|
234624
|
+
|
|
234625
|
+
// ../../backend/src/queue/workers/finalize.worker.ts
|
|
233926
234626
|
function createFinalizeWorker(deps) {
|
|
233927
234627
|
return async function processFinalizeJob(input) {
|
|
233928
234628
|
const job2 = finalizeJobSchema.parse(input);
|
|
@@ -233954,6 +234654,7 @@ function createGraphWorker(deps) {
|
|
|
233954
234654
|
}
|
|
233955
234655
|
if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
|
|
233956
234656
|
await deps.setGraphStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234657
|
+
await deps.cancelStaleRun(job2);
|
|
233957
234658
|
return;
|
|
233958
234659
|
}
|
|
233959
234660
|
if (run4.embedStatus !== "ready") {
|
|
@@ -233979,12 +234680,7 @@ function createGraphWorker(deps) {
|
|
|
233979
234680
|
if (!await deps.isCancelled?.(job2))
|
|
233980
234681
|
await deps.enqueueSummarize(job2);
|
|
233981
234682
|
} catch (error53) {
|
|
233982
|
-
|
|
233983
|
-
throw error53;
|
|
233984
|
-
await deps.setGraphStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "graph_failed");
|
|
233985
|
-
if (!await deps.isCancelled?.(job2))
|
|
233986
|
-
await deps.enqueueSummarize(job2);
|
|
233987
|
-
throw error53;
|
|
234683
|
+
await processGraphStageFailure(job2, error53, deps);
|
|
233988
234684
|
}
|
|
233989
234685
|
};
|
|
233990
234686
|
}
|
|
@@ -234055,44 +234751,7 @@ function createPrepareWorker(redisUrl, deps, options = {}) {
|
|
|
234055
234751
|
function createSummarizeWorker(deps) {
|
|
234056
234752
|
return async function processSummarizeJob(input) {
|
|
234057
234753
|
const job2 = summarizeJobSchema.parse(input);
|
|
234058
|
-
|
|
234059
|
-
return;
|
|
234060
|
-
const run4 = await deps.getRun(job2);
|
|
234061
|
-
if (!run4)
|
|
234062
|
-
throw new Error("Pipeline run not found");
|
|
234063
|
-
if (run4.ownerId !== job2.ownerId || run4.documentId !== job2.documentId) {
|
|
234064
|
-
throw new Error("Pipeline owner mismatch");
|
|
234065
|
-
}
|
|
234066
|
-
if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
|
|
234067
|
-
await deps.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234068
|
-
return;
|
|
234069
|
-
}
|
|
234070
|
-
if (!deps.enabled()) {
|
|
234071
|
-
if (await deps.isCancelled?.(job2))
|
|
234072
|
-
return;
|
|
234073
|
-
await deps.setSummaryStatus(job2.generationId, "skipped");
|
|
234074
|
-
if (!await deps.isCancelled?.(job2))
|
|
234075
|
-
await deps.enqueueFinalize(job2);
|
|
234076
|
-
return;
|
|
234077
|
-
}
|
|
234078
|
-
if (await deps.isCancelled?.(job2))
|
|
234079
|
-
return;
|
|
234080
|
-
await deps.setSummaryStatus(job2.generationId, "processing");
|
|
234081
|
-
try {
|
|
234082
|
-
if (await deps.isCancelled?.(job2))
|
|
234083
|
-
return;
|
|
234084
|
-
await deps.summarize(job2);
|
|
234085
|
-
if (await deps.isCancelled?.(job2))
|
|
234086
|
-
return;
|
|
234087
|
-
await deps.setSummaryStatus(job2.generationId, "ready");
|
|
234088
|
-
} catch (error53) {
|
|
234089
|
-
if (await deps.isCancelled?.(job2))
|
|
234090
|
-
return;
|
|
234091
|
-
await deps.setSummaryStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "summary_failed");
|
|
234092
|
-
throw error53;
|
|
234093
|
-
}
|
|
234094
|
-
if (!await deps.isCancelled?.(job2))
|
|
234095
|
-
await deps.enqueueFinalize(job2);
|
|
234754
|
+
await processSummaryStage(job2, deps);
|
|
234096
234755
|
};
|
|
234097
234756
|
}
|
|
234098
234757
|
|
|
@@ -234232,6 +234891,7 @@ var pipelineRuntime = await startRegisteredPipelineWorkers({
|
|
|
234232
234891
|
logger3.info({ legacy, recovery }, "Pipeline recovery completed");
|
|
234233
234892
|
}
|
|
234234
234893
|
});
|
|
234894
|
+
var reembedCronRuntime = startReembedCron();
|
|
234235
234895
|
ensureBucket(storage, BUCKET).catch((err) => {
|
|
234236
234896
|
logger3.error({ err }, "Failed to ensure storage bucket");
|
|
234237
234897
|
});
|
|
@@ -234286,7 +234946,7 @@ var swaggerConfig = {
|
|
|
234286
234946
|
},
|
|
234287
234947
|
info: {
|
|
234288
234948
|
title: "DocsMint API",
|
|
234289
|
-
version: "0.
|
|
234949
|
+
version: "0.6.1",
|
|
234290
234950
|
description: "Self-hosted AI-first documentation platform. Full-text + semantic search, version history, sharing, and folder organization.",
|
|
234291
234951
|
contact: { name: "HiAi-gg", url: "https://github.com/HiAi-gg/docsmint" },
|
|
234292
234952
|
license: {
|
|
@@ -234357,6 +235017,7 @@ app.listen({
|
|
|
234357
235017
|
logger3.info({ port: config3.API_PORT }, "hiai-docs API started");
|
|
234358
235018
|
var stopDocsMintApi = async () => {
|
|
234359
235019
|
logger3.info("Shutting down...");
|
|
235020
|
+
reembedCronRuntime.close();
|
|
234360
235021
|
await pipelineRuntime.close();
|
|
234361
235022
|
await app.stop();
|
|
234362
235023
|
};
|