@hiai-gg/docsmint 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -4
- package/dist/backend/index.js +753 -125
- package/dist/backend-account-runtime-cleanup.js +17 -0
- package/dist/backend-api-key-facade.js +17 -0
- package/dist/backend-pipeline-cancellation.js +17 -0
- package/dist/client.d.ts +4 -1
- package/dist/client.js +9 -0
- package/dist/lifecycle-runtime.js +16 -0
- package/dist/types.d.ts +26 -0
- package/package.json +12 -2
- package/packages/cli/src/index.ts +1 -1
- package/packages/db/src/index.ts +16 -7
- package/packages/db/src/schema.ts +34 -0
- package/packages/mcp-server/src/capabilities.ts +189 -0
- package/packages/mcp-server/src/client.ts +191 -138
- package/packages/mcp-server/src/index.ts +4 -105
- package/packages/mcp-server/src/server.ts +87 -0
- package/packages/mcp-server/src/tools/create-document.ts +19 -17
- package/packages/mcp-server/src/tools/create-folder.ts +18 -11
- package/packages/mcp-server/src/tools/create-snapshot.ts +15 -18
- package/packages/mcp-server/src/tools/export-document.ts +12 -12
- package/packages/mcp-server/src/tools/get-document.ts +12 -10
- package/packages/mcp-server/src/tools/list-documents.ts +15 -23
- package/packages/mcp-server/src/tools/list-folders.ts +12 -11
- package/packages/mcp-server/src/tools/search.ts +20 -23
- package/packages/mcp-server/src/tools/update-document.ts +22 -14
- package/packages/mcp-server/src/tools/version-history.ts +13 -18
- package/server.json +55 -0
- package/skills/docsmint-document-manager/SKILL.md +33 -0
package/dist/backend/index.js
CHANGED
|
@@ -42744,6 +42744,7 @@ __export(exports_schema, {
|
|
|
42744
42744
|
documentRelations: () => documentRelations,
|
|
42745
42745
|
documentPipelineRuns: () => documentPipelineRuns,
|
|
42746
42746
|
documentPipelineBatches: () => documentPipelineBatches,
|
|
42747
|
+
documentKnowledgeSummaries: () => documentKnowledgeSummaries,
|
|
42747
42748
|
documentEmbeddings: () => documentEmbeddings,
|
|
42748
42749
|
documentEmbeddingRelations: () => documentEmbeddingRelations,
|
|
42749
42750
|
documentCreateOperations: () => documentCreateOperations,
|
|
@@ -42756,7 +42757,7 @@ __export(exports_schema, {
|
|
|
42756
42757
|
apiKeyRelations: () => apiKeyRelations,
|
|
42757
42758
|
accounts: () => accounts
|
|
42758
42759
|
});
|
|
42759
|
-
var vector4, tsvector, documentVisibilityEnum, shareRoleEnum, shareAccessModeEnum, shareGrantStatusEnum, embeddingStatusEnum, pipelineStageEnum, pipelineStatusEnum, lifecycleOperationKindEnum, lifecycleOperationStatusEnum, users, sessions, accounts, verifications, folders, folderRelations, documents, documentRelations, documentPipelineRuns, documentPipelineBatches, tags, tagRelations, categories, categoryRelations, documentTags, documentTagRelations, shareLinks, shareLinkRelations, guestAccess, guestAccessRelations, attachments, attachmentRelations, versions, documentEmbeddings, documentEmbeddingRelations, apiKeys, apiKeyRelations, auditLog, lifecycleOperations, documentCreateOperations, versionRelations;
|
|
42760
|
+
var vector4, tsvector, documentVisibilityEnum, shareRoleEnum, shareAccessModeEnum, shareGrantStatusEnum, embeddingStatusEnum, pipelineStageEnum, pipelineStatusEnum, lifecycleOperationKindEnum, lifecycleOperationStatusEnum, users, sessions, accounts, verifications, folders, folderRelations, documents, documentRelations, documentPipelineRuns, documentPipelineBatches, documentKnowledgeSummaries, tags, tagRelations, categories, categoryRelations, documentTags, documentTagRelations, shareLinks, shareLinkRelations, guestAccess, guestAccessRelations, attachments, attachmentRelations, versions, documentEmbeddings, documentEmbeddingRelations, apiKeys, apiKeyRelations, auditLog, lifecycleOperations, documentCreateOperations, versionRelations;
|
|
42760
42761
|
var init_schema2 = __esm(() => {
|
|
42761
42762
|
init_pg_core();
|
|
42762
42763
|
init_drizzle_orm();
|
|
@@ -43010,6 +43011,22 @@ var init_schema2 = __esm(() => {
|
|
|
43010
43011
|
index("document_pipeline_batches_stage_status_available_idx").on(table3.stage, table3.status, table3.availableAt),
|
|
43011
43012
|
index("document_pipeline_batches_document_id_idx").on(table3.documentId)
|
|
43012
43013
|
]);
|
|
43014
|
+
documentKnowledgeSummaries = pgTable("document_knowledge_summaries", {
|
|
43015
|
+
id: uuid().primaryKey().defaultRandom(),
|
|
43016
|
+
document_id: uuid().notNull().references(() => documents.id, { onDelete: "cascade" }),
|
|
43017
|
+
owner_id: uuid().notNull().references(() => users.id, { onDelete: "cascade" }),
|
|
43018
|
+
workspace_id: text(),
|
|
43019
|
+
generation_id: uuid().notNull(),
|
|
43020
|
+
revision: text().notNull(),
|
|
43021
|
+
language: text().notNull(),
|
|
43022
|
+
description: text().notNull(),
|
|
43023
|
+
keywords: text().array().notNull().default(sql`ARRAY[]::text[]`),
|
|
43024
|
+
created_at: timestamp().defaultNow().notNull(),
|
|
43025
|
+
updated_at: timestamp().defaultNow().notNull()
|
|
43026
|
+
}, (table3) => [
|
|
43027
|
+
uniqueIndex("document_knowledge_summaries_document_generation_idx").on(table3.document_id, table3.generation_id),
|
|
43028
|
+
index("document_knowledge_summaries_owner_document_idx").on(table3.owner_id, table3.document_id)
|
|
43029
|
+
]);
|
|
43013
43030
|
tags = pgTable("tags", {
|
|
43014
43031
|
id: uuid("id").primaryKey().defaultRandom(),
|
|
43015
43032
|
ownerId: uuid("owner_id").notNull().references(() => users.id, { onDelete: "cascade" }),
|
|
@@ -173203,6 +173220,8 @@ async function validateApiKey(key) {
|
|
|
173203
173220
|
scopes
|
|
173204
173221
|
};
|
|
173205
173222
|
}
|
|
173223
|
+
// ../db/src/document-writer.ts
|
|
173224
|
+
init_schema2();
|
|
173206
173225
|
|
|
173207
173226
|
// ../db/src/index.ts
|
|
173208
173227
|
init_schema2();
|
|
@@ -194094,6 +194113,7 @@ function contentAccessForExternalContext(ctx) {
|
|
|
194094
194113
|
permissions.add("write");
|
|
194095
194114
|
} else if (ctx.actorRole === "editor") {
|
|
194096
194115
|
permissions.add("edit");
|
|
194116
|
+
permissions.add("write");
|
|
194097
194117
|
}
|
|
194098
194118
|
return {
|
|
194099
194119
|
principal: { kind: "session", userId: ctx.userId },
|
|
@@ -194144,6 +194164,9 @@ function canAccessContent(access, action) {
|
|
|
194144
194164
|
return access.permissions.has(action);
|
|
194145
194165
|
return !access.restricted || access.permissions.has(action);
|
|
194146
194166
|
}
|
|
194167
|
+
function canManageCategories(access) {
|
|
194168
|
+
return !access.restricted && canAccessContent(access, "write");
|
|
194169
|
+
}
|
|
194147
194170
|
function effectiveDocumentCategory(row) {
|
|
194148
194171
|
return row.categoryId ?? row.folderCategoryId ?? null;
|
|
194149
194172
|
}
|
|
@@ -194237,7 +194260,8 @@ var enqueueDocumentPipelineSchema = exports_external2.object({
|
|
|
194237
194260
|
workspaceId: exports_external2.string().min(1).optional(),
|
|
194238
194261
|
revision: exports_external2.string().min(1),
|
|
194239
194262
|
source: pipelineSourceSchema,
|
|
194240
|
-
requestedAt: exports_external2.iso.datetime().optional()
|
|
194263
|
+
requestedAt: exports_external2.iso.datetime().optional(),
|
|
194264
|
+
forceNewGeneration: exports_external2.boolean().optional()
|
|
194241
194265
|
});
|
|
194242
194266
|
var JOB_IDS = {
|
|
194243
194267
|
prepare: (documentId, generationId, workspaceId) => `prepare-${documentId}-${generationId}${workspaceId ? `-${workspaceId}` : ""}`,
|
|
@@ -194266,7 +194290,14 @@ var postgresRunStore = {
|
|
|
194266
194290
|
const [document2] = await tx.select({ id: documents.id }).from(documents).where(and(eq(documents.id, input.documentId), ownerBoundary)).limit(1);
|
|
194267
194291
|
if (!document2)
|
|
194268
194292
|
throw new Error("Document not found for pipeline owner");
|
|
194269
|
-
|
|
194293
|
+
if (input.forceNewGeneration) {
|
|
194294
|
+
await tx.update(documentPipelineRuns).set({
|
|
194295
|
+
status: "cancelled",
|
|
194296
|
+
errorCode: "superseded_by_reindex",
|
|
194297
|
+
updatedAt: new Date
|
|
194298
|
+
}).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES])));
|
|
194299
|
+
}
|
|
194300
|
+
const [existing] = input.forceNewGeneration ? [] : await tx.select({ generationId: documentPipelineRuns.generationId }).from(documentPipelineRuns).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, eq(documentPipelineRuns.revision, input.revision), inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES]))).limit(1);
|
|
194270
194301
|
if (existing)
|
|
194271
194302
|
return { run: existing, created: false };
|
|
194272
194303
|
const [created] = await tx.insert(documentPipelineRuns).values({
|
|
@@ -194302,7 +194333,8 @@ async function enqueueDocumentPipeline(input, dependencies) {
|
|
|
194302
194333
|
const { run, created } = await deps.runs.findOrCreate({
|
|
194303
194334
|
...parsed,
|
|
194304
194335
|
requestedAt,
|
|
194305
|
-
generationId: proposedGenerationId
|
|
194336
|
+
generationId: proposedGenerationId,
|
|
194337
|
+
forceNewGeneration: parsed.forceNewGeneration
|
|
194306
194338
|
});
|
|
194307
194339
|
if (created) {
|
|
194308
194340
|
const job2 = {
|
|
@@ -194373,10 +194405,10 @@ ${content52}`;
|
|
|
194373
194405
|
init_logger3();
|
|
194374
194406
|
init_redis();
|
|
194375
194407
|
var LEGACY_EMBEDDING_QUEUE_KEY = "hiai-docs:embedding-queue";
|
|
194376
|
-
async function enqueueEmbedding(documentId, source = "interactive") {
|
|
194408
|
+
async function enqueueEmbedding(documentId, source = "interactive", workspaceId, options = {}) {
|
|
194377
194409
|
let pipelineInput;
|
|
194378
194410
|
try {
|
|
194379
|
-
await withTenant(adminTenantContext(ZERO_UUID), async (tx) => {
|
|
194411
|
+
await withTenant({ ...adminTenantContext(ZERO_UUID), workspaceId }, async (tx) => {
|
|
194380
194412
|
const rows = await tx.select({
|
|
194381
194413
|
active: documents.activeEmbeddingGeneration,
|
|
194382
194414
|
ownerId: documents.ownerId,
|
|
@@ -194387,7 +194419,7 @@ async function enqueueEmbedding(documentId, source = "interactive") {
|
|
|
194387
194419
|
if (!document2)
|
|
194388
194420
|
return;
|
|
194389
194421
|
const revision = contentHash(document2.title, document2.content ?? "");
|
|
194390
|
-
pipelineInput = { ownerId: document2.ownerId, revision };
|
|
194422
|
+
pipelineInput = { ownerId: document2.ownerId, revision, workspaceId };
|
|
194391
194423
|
const activeGeneration = document2.active;
|
|
194392
194424
|
if (activeGeneration) {
|
|
194393
194425
|
await tx.update(documents).set({
|
|
@@ -194410,8 +194442,10 @@ async function enqueueEmbedding(documentId, source = "interactive") {
|
|
|
194410
194442
|
await enqueueDocumentPipeline({
|
|
194411
194443
|
documentId,
|
|
194412
194444
|
ownerId: pipelineInput.ownerId,
|
|
194445
|
+
workspaceId: pipelineInput.workspaceId,
|
|
194413
194446
|
revision: pipelineInput.revision,
|
|
194414
|
-
source
|
|
194447
|
+
source,
|
|
194448
|
+
forceNewGeneration: options.forceNewGeneration
|
|
194415
194449
|
});
|
|
194416
194450
|
return true;
|
|
194417
194451
|
} catch (err) {
|
|
@@ -194449,7 +194483,7 @@ async function enqueueReembed(docIds, workspaceId) {
|
|
|
194449
194483
|
let pushed = 0;
|
|
194450
194484
|
for (const id2 of unique) {
|
|
194451
194485
|
if (await claimEnqueueSlot(id2, workspaceId)) {
|
|
194452
|
-
enqueueEmbedding(id2);
|
|
194486
|
+
enqueueEmbedding(id2, "interactive", workspaceId);
|
|
194453
194487
|
pushed += 1;
|
|
194454
194488
|
}
|
|
194455
194489
|
}
|
|
@@ -196359,9 +196393,6 @@ init_schema2();
|
|
|
196359
196393
|
init_drizzle_orm();
|
|
196360
196394
|
init_zod();
|
|
196361
196395
|
init_logger3();
|
|
196362
|
-
function isCategoryManager(principal) {
|
|
196363
|
-
return principal?.kind === "session" || principal?.kind === "operator";
|
|
196364
|
-
}
|
|
196365
196396
|
var createCategorySchema = exports_external2.object({
|
|
196366
196397
|
name: exports_external2.string().trim().min(1).max(255),
|
|
196367
196398
|
apiMode: exports_external2.enum(["unavailable", "global", "category"]).optional(),
|
|
@@ -196499,9 +196530,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
|
|
|
196499
196530
|
set2.status = 401;
|
|
196500
196531
|
return { error: "Unauthorized" };
|
|
196501
196532
|
}
|
|
196502
|
-
if (!
|
|
196533
|
+
if (!canManageCategories(access)) {
|
|
196503
196534
|
set2.status = 403;
|
|
196504
|
-
return { error: "
|
|
196535
|
+
return { error: "Full workspace write access required" };
|
|
196505
196536
|
}
|
|
196506
196537
|
const userId = ctx.userId;
|
|
196507
196538
|
const parsed = createCategorySchema.safeParse(await request.json());
|
|
@@ -196551,9 +196582,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
|
|
|
196551
196582
|
set2.status = 401;
|
|
196552
196583
|
return { error: "Unauthorized" };
|
|
196553
196584
|
}
|
|
196554
|
-
if (!
|
|
196585
|
+
if (!canManageCategories(access)) {
|
|
196555
196586
|
set2.status = 403;
|
|
196556
|
-
return { error: "
|
|
196587
|
+
return { error: "Full workspace write access required" };
|
|
196557
196588
|
}
|
|
196558
196589
|
const userId = ctx.userId;
|
|
196559
196590
|
const parsed = updateCategorySchema.safeParse(await request.json());
|
|
@@ -196622,9 +196653,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
|
|
|
196622
196653
|
set2.status = 401;
|
|
196623
196654
|
return { error: "Unauthorized" };
|
|
196624
196655
|
}
|
|
196625
|
-
if (!
|
|
196656
|
+
if (!canManageCategories(access)) {
|
|
196626
196657
|
set2.status = 403;
|
|
196627
|
-
return { error: "
|
|
196658
|
+
return { error: "Full workspace write access required" };
|
|
196628
196659
|
}
|
|
196629
196660
|
const userId = ctx.userId;
|
|
196630
196661
|
try {
|
|
@@ -227123,6 +227154,7 @@ var cursorListQuerySchema = exports_external2.object({
|
|
|
227123
227154
|
sortBy: exports_external2.enum(["title", "category", "folder", "updated"]).default("updated"),
|
|
227124
227155
|
sortOrder: exports_external2.enum(["asc", "desc"]).default("desc")
|
|
227125
227156
|
});
|
|
227157
|
+
var documentIdParamsSchema = exports_external2.object({ id: exports_external2.string().uuid() });
|
|
227126
227158
|
function cursorScopeHash(input) {
|
|
227127
227159
|
return Buffer.from(new Bun.CryptoHasher("sha256").update(input).digest("hex"), "utf8").toString("base64url");
|
|
227128
227160
|
}
|
|
@@ -227641,6 +227673,180 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
|
|
|
227641
227673
|
set2.status = 500;
|
|
227642
227674
|
return { error: "Failed to load pipeline progress" };
|
|
227643
227675
|
}
|
|
227676
|
+
}).get("/documents/:id/knowledge-summary", async ({ params: params2, set: set2, request }) => {
|
|
227677
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
227678
|
+
if (!parsedParams.success) {
|
|
227679
|
+
set2.status = 400;
|
|
227680
|
+
return { error: "Invalid document id" };
|
|
227681
|
+
}
|
|
227682
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227683
|
+
const rl = await documentRateLimiter(ip, request);
|
|
227684
|
+
if (!rl.allowed) {
|
|
227685
|
+
set2.status = 429;
|
|
227686
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
227687
|
+
return { error: "Too many requests" };
|
|
227688
|
+
}
|
|
227689
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
227690
|
+
const access = await resolveContentAccess(request);
|
|
227691
|
+
const ctx = access.ctx;
|
|
227692
|
+
if (ctx.role === "none") {
|
|
227693
|
+
set2.status = 401;
|
|
227694
|
+
return { error: "Unauthorized" };
|
|
227695
|
+
}
|
|
227696
|
+
if (!canAccessContent(access, "read")) {
|
|
227697
|
+
set2.status = 403;
|
|
227698
|
+
return { error: "Forbidden" };
|
|
227699
|
+
}
|
|
227700
|
+
const [summary] = await withTenant(ctx, (tx) => tx.select({
|
|
227701
|
+
documentId: documentKnowledgeSummaries.document_id,
|
|
227702
|
+
generationId: documentKnowledgeSummaries.generation_id,
|
|
227703
|
+
revision: documentKnowledgeSummaries.revision,
|
|
227704
|
+
language: documentKnowledgeSummaries.language,
|
|
227705
|
+
description: documentKnowledgeSummaries.description,
|
|
227706
|
+
keywords: documentKnowledgeSummaries.keywords,
|
|
227707
|
+
createdAt: documentKnowledgeSummaries.created_at,
|
|
227708
|
+
updatedAt: documentKnowledgeSummaries.updated_at
|
|
227709
|
+
}).from(documentKnowledgeSummaries).innerJoin(documents, eq(documents.id, documentKnowledgeSummaries.document_id)).where(and(eq(documentKnowledgeSummaries.document_id, parsedParams.data.id), eq(documentKnowledgeSummaries.generation_id, documents.activeEmbeddingGeneration), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1));
|
|
227710
|
+
if (!summary) {
|
|
227711
|
+
set2.status = 404;
|
|
227712
|
+
return { error: "Knowledge summary not found" };
|
|
227713
|
+
}
|
|
227714
|
+
return summary;
|
|
227715
|
+
}).get("/documents/:id/index-status", async ({ params: params2, set: set2, request }) => {
|
|
227716
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
227717
|
+
if (!parsedParams.success) {
|
|
227718
|
+
set2.status = 400;
|
|
227719
|
+
return { error: "Invalid document id" };
|
|
227720
|
+
}
|
|
227721
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227722
|
+
const rl = await documentRateLimiter(ip, request);
|
|
227723
|
+
if (!rl.allowed) {
|
|
227724
|
+
set2.status = 429;
|
|
227725
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
227726
|
+
return { error: "Too many requests" };
|
|
227727
|
+
}
|
|
227728
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
227729
|
+
const access = await resolveContentAccess(request);
|
|
227730
|
+
const ctx = access.ctx;
|
|
227731
|
+
if (ctx.role === "none") {
|
|
227732
|
+
set2.status = 401;
|
|
227733
|
+
return { error: "Unauthorized" };
|
|
227734
|
+
}
|
|
227735
|
+
if (!canAccessContent(access, "read")) {
|
|
227736
|
+
set2.status = 403;
|
|
227737
|
+
return { error: "Forbidden" };
|
|
227738
|
+
}
|
|
227739
|
+
const result = await withTenant(ctx, async (tx) => {
|
|
227740
|
+
const [document2] = await tx.select({
|
|
227741
|
+
id: documents.id,
|
|
227742
|
+
embeddingStatus: documents.embeddingStatus,
|
|
227743
|
+
activeGenerationId: documents.activeEmbeddingGeneration,
|
|
227744
|
+
pendingGenerationId: documents.pendingEmbeddingGeneration,
|
|
227745
|
+
embeddingProfile: documents.embeddingProfile,
|
|
227746
|
+
embeddingErrorCode: documents.embeddingErrorCode,
|
|
227747
|
+
embeddingUpdatedAt: documents.embeddingUpdatedAt
|
|
227748
|
+
}).from(documents).where(and(eq(documents.id, parsedParams.data.id), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1);
|
|
227749
|
+
if (!document2)
|
|
227750
|
+
return null;
|
|
227751
|
+
const active = document2.activeGenerationId ? await tx.select({ id: documentEmbeddings.id }).from(documentEmbeddings).where(and(eq(documentEmbeddings.documentId, document2.id), eq(documentEmbeddings.generationId, document2.activeGenerationId), eq(documentEmbeddings.isValid, true), eq(documentEmbeddings.embeddingDimensions, 1024), eq(documentEmbeddings.embeddingProfile, document2.embeddingProfile ?? ""))).limit(1) : [];
|
|
227752
|
+
const [run5] = await tx.select({
|
|
227753
|
+
documentId: documentPipelineRuns.documentId,
|
|
227754
|
+
generationId: documentPipelineRuns.generationId,
|
|
227755
|
+
revision: documentPipelineRuns.revision,
|
|
227756
|
+
status: documentPipelineRuns.status,
|
|
227757
|
+
prepareStatus: documentPipelineRuns.prepareStatus,
|
|
227758
|
+
embedStatus: documentPipelineRuns.embedStatus,
|
|
227759
|
+
graphStatus: documentPipelineRuns.graphStatus,
|
|
227760
|
+
summarizeStatus: documentPipelineRuns.summarizeStatus,
|
|
227761
|
+
finalizeStatus: documentPipelineRuns.finalizeStatus,
|
|
227762
|
+
totalBatches: documentPipelineRuns.totalBatches,
|
|
227763
|
+
completedBatches: documentPipelineRuns.completedBatches,
|
|
227764
|
+
failedBatches: documentPipelineRuns.failedBatches,
|
|
227765
|
+
updatedAt: documentPipelineRuns.updatedAt
|
|
227766
|
+
}).from(documentPipelineRuns).where(eq(documentPipelineRuns.documentId, document2.id)).orderBy(desc(documentPipelineRuns.updatedAt)).limit(1);
|
|
227767
|
+
return {
|
|
227768
|
+
document: document2,
|
|
227769
|
+
searchable: active.length > 0,
|
|
227770
|
+
run: run5
|
|
227771
|
+
};
|
|
227772
|
+
});
|
|
227773
|
+
if (!result) {
|
|
227774
|
+
set2.status = 404;
|
|
227775
|
+
return { error: "Document not found" };
|
|
227776
|
+
}
|
|
227777
|
+
const run4 = result.run;
|
|
227778
|
+
return {
|
|
227779
|
+
documentId: result.document.id,
|
|
227780
|
+
embeddingStatus: result.document.embeddingStatus,
|
|
227781
|
+
activeGenerationId: result.document.activeGenerationId,
|
|
227782
|
+
pendingGenerationId: result.document.pendingGenerationId,
|
|
227783
|
+
embeddingProfile: result.document.embeddingProfile,
|
|
227784
|
+
embeddingErrorCode: result.document.embeddingErrorCode,
|
|
227785
|
+
embeddingUpdatedAt: result.document.embeddingUpdatedAt,
|
|
227786
|
+
searchable: result.searchable,
|
|
227787
|
+
pipeline: run4 ? {
|
|
227788
|
+
documentId: run4.documentId,
|
|
227789
|
+
generationId: run4.generationId,
|
|
227790
|
+
status: run4.status,
|
|
227791
|
+
revision: run4.revision,
|
|
227792
|
+
stages: {
|
|
227793
|
+
prepare: run4.prepareStatus,
|
|
227794
|
+
embed: run4.embedStatus,
|
|
227795
|
+
graph: run4.graphStatus,
|
|
227796
|
+
summarize: run4.summarizeStatus,
|
|
227797
|
+
finalize: run4.finalizeStatus
|
|
227798
|
+
},
|
|
227799
|
+
batches: {
|
|
227800
|
+
total: run4.totalBatches,
|
|
227801
|
+
completed: run4.completedBatches,
|
|
227802
|
+
failed: run4.failedBatches
|
|
227803
|
+
},
|
|
227804
|
+
updatedAt: run4.updatedAt
|
|
227805
|
+
} : null
|
|
227806
|
+
};
|
|
227807
|
+
}).post("/documents/:id/index/refresh", async ({ params: params2, set: set2, request }) => {
|
|
227808
|
+
const parsedParams = documentIdParamsSchema.safeParse(params2);
|
|
227809
|
+
if (!parsedParams.success) {
|
|
227810
|
+
set2.status = 400;
|
|
227811
|
+
return { error: "Invalid document id" };
|
|
227812
|
+
}
|
|
227813
|
+
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227814
|
+
const rl = await writeRateLimiter(ip, request);
|
|
227815
|
+
if (!rl.allowed) {
|
|
227816
|
+
set2.status = 429;
|
|
227817
|
+
set2.headers = rateLimitHeaders(0, rl.retryAfter);
|
|
227818
|
+
return { error: "Too many requests" };
|
|
227819
|
+
}
|
|
227820
|
+
set2.headers = rateLimitHeaders(rl.remaining);
|
|
227821
|
+
const access = await resolveContentAccess(request);
|
|
227822
|
+
const ctx = access.ctx;
|
|
227823
|
+
if (ctx.role === "none") {
|
|
227824
|
+
set2.status = 401;
|
|
227825
|
+
return { error: "Unauthorized" };
|
|
227826
|
+
}
|
|
227827
|
+
if (!canAccessContent(access, "edit")) {
|
|
227828
|
+
set2.status = 403;
|
|
227829
|
+
return { error: "Forbidden" };
|
|
227830
|
+
}
|
|
227831
|
+
const [document2] = await withTenant(ctx, (tx) => tx.select({
|
|
227832
|
+
id: documents.id,
|
|
227833
|
+
ownerId: documents.ownerId,
|
|
227834
|
+
title: documents.title,
|
|
227835
|
+
content: documents.content
|
|
227836
|
+
}).from(documents).where(and(eq(documents.id, parsedParams.data.id), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1));
|
|
227837
|
+
if (!document2) {
|
|
227838
|
+
set2.status = 404;
|
|
227839
|
+
return { error: "Document not found" };
|
|
227840
|
+
}
|
|
227841
|
+
const queued = await enqueueDocumentPipeline({
|
|
227842
|
+
documentId: document2.id,
|
|
227843
|
+
ownerId: document2.ownerId,
|
|
227844
|
+
workspaceId: ctx.workspaceId,
|
|
227845
|
+
revision: contentHash(document2.title, document2.content ?? ""),
|
|
227846
|
+
source: "api"
|
|
227847
|
+
});
|
|
227848
|
+
set2.status = 202;
|
|
227849
|
+
return { documentId: document2.id, ...queued };
|
|
227644
227850
|
}).get("/documents/:id", async ({ params: params2, set: set2, request }) => {
|
|
227645
227851
|
const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
|
|
227646
227852
|
const rl = await documentRateLimiter(ip, request);
|
|
@@ -229073,14 +229279,15 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
|
|
|
229073
229279
|
set2.status = 400;
|
|
229074
229280
|
return { error: "Invalid query", details: parsed.error.flatten() };
|
|
229075
229281
|
}
|
|
229076
|
-
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229077
|
-
return { entities: [] };
|
|
229078
|
-
}
|
|
229079
229282
|
try {
|
|
229080
229283
|
const allowedIds = await allowedGraphDocumentIds(access, [
|
|
229081
229284
|
parsed.data.docId
|
|
229082
229285
|
]);
|
|
229083
|
-
if (!allowedIds.has(parsed.data.docId))
|
|
229286
|
+
if (!allowedIds.has(parsed.data.docId)) {
|
|
229287
|
+
set2.status = 404;
|
|
229288
|
+
return { error: "Document not found" };
|
|
229289
|
+
}
|
|
229290
|
+
if (!config3.GRAPH_SEARCH_ENABLED)
|
|
229084
229291
|
return { entities: [] };
|
|
229085
229292
|
const entities = await fetchDocumentEntities(parsed.data.docId);
|
|
229086
229293
|
return { entities };
|
|
@@ -229112,14 +229319,15 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
|
|
|
229112
229319
|
set2.status = 400;
|
|
229113
229320
|
return { error: "Invalid params", details: parsed.error.flatten() };
|
|
229114
229321
|
}
|
|
229115
|
-
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229116
|
-
return { related: [] };
|
|
229117
|
-
}
|
|
229118
229322
|
try {
|
|
229119
229323
|
const seedIds = await allowedGraphDocumentIds(access, [
|
|
229120
229324
|
parsed.data.docId
|
|
229121
229325
|
]);
|
|
229122
|
-
if (!seedIds.has(parsed.data.docId))
|
|
229326
|
+
if (!seedIds.has(parsed.data.docId)) {
|
|
229327
|
+
set2.status = 404;
|
|
229328
|
+
return { error: "Document not found" };
|
|
229329
|
+
}
|
|
229330
|
+
if (!config3.GRAPH_SEARCH_ENABLED)
|
|
229123
229331
|
return { related: [] };
|
|
229124
229332
|
const related = await fetchRelatedDocuments(ctx, parsed.data.docId, access);
|
|
229125
229333
|
return { related };
|
|
@@ -229152,12 +229360,16 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
|
|
|
229152
229360
|
return { error: "Invalid body", details: parsed.error.flatten() };
|
|
229153
229361
|
}
|
|
229154
229362
|
const { query, docIds, maxResults } = parsed.data;
|
|
229155
|
-
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229156
|
-
return { query, entities: [], relatedDocs: [] };
|
|
229157
|
-
}
|
|
229158
229363
|
try {
|
|
229159
229364
|
const authorizedSeeds = await allowedGraphDocumentIds(access, docIds);
|
|
229160
|
-
|
|
229365
|
+
if (docIds.some((id3) => !authorizedSeeds.has(id3))) {
|
|
229366
|
+
set2.status = 404;
|
|
229367
|
+
return { error: "One or more documents were not found" };
|
|
229368
|
+
}
|
|
229369
|
+
if (!config3.GRAPH_SEARCH_ENABLED) {
|
|
229370
|
+
return { query, entities: [], relatedDocs: [] };
|
|
229371
|
+
}
|
|
229372
|
+
const result = await graphRagLookup(ctx, docIds, maxResults, access);
|
|
229161
229373
|
return { query, ...result };
|
|
229162
229374
|
} catch (err) {
|
|
229163
229375
|
logger3.warn({ err, docIds: docIds.length }, "Graph RAG search failed \u2014 returning empty");
|
|
@@ -230229,7 +230441,6 @@ async function retrieveVector(ctx, plan, limit, minimum, chunkLimit, execute, pr
|
|
|
230229
230441
|
JOIN documents d ON d.id = de.document_id
|
|
230230
230442
|
WHERE d.owner_id = ${ctx.userId}
|
|
230231
230443
|
${scope}
|
|
230232
|
-
AND d.embedding_status = 'ready'
|
|
230233
230444
|
AND d.active_embedding_generation IS NOT NULL
|
|
230234
230445
|
AND de.generation_id = d.active_embedding_generation
|
|
230235
230446
|
AND de.is_valid = true
|
|
@@ -230961,7 +231172,6 @@ async function hydrateResults(ctx, items, includeChunks, query, allowedDocumentI
|
|
|
230961
231172
|
JOIN documents d ON d.id = de.document_id
|
|
230962
231173
|
WHERE de.document_id IN (${sql.join(ids.map((id3) => sql`${id3}`), sql`, `)})
|
|
230963
231174
|
AND ${tenantOwnerSql("d", ctx)}
|
|
230964
|
-
AND d.embedding_status = 'ready'
|
|
230965
231175
|
AND d.active_embedding_generation IS NOT NULL
|
|
230966
231176
|
AND de.generation_id = d.active_embedding_generation
|
|
230967
231177
|
AND de.is_valid = true
|
|
@@ -232729,6 +232939,116 @@ async function ensureApiKeyOwner() {
|
|
|
232729
232939
|
init_config();
|
|
232730
232940
|
init_logger3();
|
|
232731
232941
|
|
|
232942
|
+
// ../../backend/src/lib/reembed-cron.ts
|
|
232943
|
+
init_schema2();
|
|
232944
|
+
init_drizzle_orm();
|
|
232945
|
+
init_config();
|
|
232946
|
+
|
|
232947
|
+
// ../../backend/src/lib/cron-timer-registry.ts
|
|
232948
|
+
function createCronTimerRegistry(clear = clearInterval) {
|
|
232949
|
+
const handles = [];
|
|
232950
|
+
let closed = false;
|
|
232951
|
+
return {
|
|
232952
|
+
register(handle) {
|
|
232953
|
+
if (closed) {
|
|
232954
|
+
clear(handle);
|
|
232955
|
+
return;
|
|
232956
|
+
}
|
|
232957
|
+
handles.push(handle);
|
|
232958
|
+
},
|
|
232959
|
+
close() {
|
|
232960
|
+
if (closed)
|
|
232961
|
+
return;
|
|
232962
|
+
closed = true;
|
|
232963
|
+
for (const handle of handles)
|
|
232964
|
+
clear(handle);
|
|
232965
|
+
handles.length = 0;
|
|
232966
|
+
}
|
|
232967
|
+
};
|
|
232968
|
+
}
|
|
232969
|
+
|
|
232970
|
+
// ../../backend/src/lib/reembed-cron.ts
|
|
232971
|
+
init_logger3();
|
|
232972
|
+
var METADATA_DEBOUNCE_MINUTES = 3;
|
|
232973
|
+
var CRON_BATCH_SIZE = 100;
|
|
232974
|
+
var CRON_TENANT = adminTenantContext(ZERO_UUID);
|
|
232975
|
+
function startReembedCron() {
|
|
232976
|
+
const timers = createCronTimerRegistry();
|
|
232977
|
+
const metadataIntervalMs = config3.METADATA_REEMBED_CRON_INTERVAL_MINUTES * 60 * 1000;
|
|
232978
|
+
if (metadataIntervalMs > 0) {
|
|
232979
|
+
logger3.info({ intervalMinutes: config3.METADATA_REEMBED_CRON_INTERVAL_MINUTES }, "Reembed metadata-stale cron started");
|
|
232980
|
+
processStaleMetadataChanges().catch((err) => {
|
|
232981
|
+
logger3.error({ err }, "Reembed metadata-stale cron initial run failed");
|
|
232982
|
+
});
|
|
232983
|
+
timers.register(setInterval(() => {
|
|
232984
|
+
processStaleMetadataChanges().catch((err) => {
|
|
232985
|
+
logger3.error({ err }, "Reembed metadata-stale cron tick failed");
|
|
232986
|
+
});
|
|
232987
|
+
}, metadataIntervalMs));
|
|
232988
|
+
} else {
|
|
232989
|
+
logger3.warn("METADATA_REEMBED_CRON_INTERVAL_MINUTES=0 \u2014 metadata-stale cron is DISABLED. " + "Metadata-only changes will NOT trigger re-embed until the cron is re-enabled.");
|
|
232990
|
+
}
|
|
232991
|
+
const idleIntervalMs = config3.REEMBED_CRON_INTERVAL_MINUTES * 60 * 1000;
|
|
232992
|
+
if (idleIntervalMs > 0) {
|
|
232993
|
+
logger3.info({ intervalMinutes: config3.REEMBED_CRON_INTERVAL_MINUTES }, "Reembed idle-pending cron started");
|
|
232994
|
+
processIdlePendingChanges().catch((err) => {
|
|
232995
|
+
logger3.error({ err }, "Reembed idle-pending cron initial run failed");
|
|
232996
|
+
});
|
|
232997
|
+
timers.register(setInterval(() => {
|
|
232998
|
+
processIdlePendingChanges().catch((err) => {
|
|
232999
|
+
logger3.error({ err }, "Reembed idle-pending cron tick failed");
|
|
233000
|
+
});
|
|
233001
|
+
}, idleIntervalMs));
|
|
233002
|
+
} else {
|
|
233003
|
+
logger3.warn("REEMBED_CRON_INTERVAL_MINUTES=0 \u2014 idle-pending cron is DISABLED. " + "Sub-threshold edits will NOT trigger catch-up embeds until the cron is re-enabled.");
|
|
233004
|
+
}
|
|
233005
|
+
return { close: () => timers.close() };
|
|
233006
|
+
}
|
|
233007
|
+
async function processStaleMetadataChanges() {
|
|
233008
|
+
const cutoff = new Date(Date.now() - METADATA_DEBOUNCE_MINUTES * 60 * 1000);
|
|
233009
|
+
const stale = await withTenant(CRON_TENANT, (tx) => tx.select({
|
|
233010
|
+
id: documents.id,
|
|
233011
|
+
workspaceId: documents.workspaceId,
|
|
233012
|
+
metadataChangedAt: documents.metadataChangedAt
|
|
233013
|
+
}).from(documents).where(and(isNotNull(documents.metadataChangedAt), lt(documents.metadataChangedAt, cutoff))).orderBy(documents.metadataChangedAt).limit(CRON_BATCH_SIZE));
|
|
233014
|
+
if (stale.length === 0)
|
|
233015
|
+
return;
|
|
233016
|
+
logger3.debug({ count: stale.length }, "Reembed cron: processing stale metadata changes");
|
|
233017
|
+
let enqueued = 0;
|
|
233018
|
+
for (const row of stale) {
|
|
233019
|
+
const originalTimestamp = row.metadataChangedAt instanceof Date ? row.metadataChangedAt.toISOString() : String(row.metadataChangedAt);
|
|
233020
|
+
const cleared = await withTenant(CRON_TENANT, (tx) => tx.execute(sql`
|
|
233021
|
+
UPDATE documents
|
|
233022
|
+
SET metadata_changed_at = NULL
|
|
233023
|
+
WHERE id = ${row.id}
|
|
233024
|
+
AND metadata_changed_at = ${originalTimestamp}
|
|
233025
|
+
RETURNING id
|
|
233026
|
+
`));
|
|
233027
|
+
if (Array.isArray(cleared) && cleared.length > 0) {
|
|
233028
|
+
await enqueueReembed([row.id], row.workspaceId ?? undefined);
|
|
233029
|
+
enqueued += 1;
|
|
233030
|
+
}
|
|
233031
|
+
}
|
|
233032
|
+
if (enqueued > 0) {
|
|
233033
|
+
logger3.info({ scanned: stale.length, enqueued }, "Reembed cron: enqueued stale metadata changes");
|
|
233034
|
+
}
|
|
233035
|
+
}
|
|
233036
|
+
async function processIdlePendingChanges() {
|
|
233037
|
+
const cutoff = new Date(Date.now() - config3.REEMBED_MAX_IDLE_HOURS * 3600000);
|
|
233038
|
+
const idle = await withTenant(CRON_TENANT, (tx) => tx.select({ id: documents.id, workspaceId: documents.workspaceId }).from(documents).where(and(eq(documents.pendingMinorChanges, true), lt(documents.lastSignificantUpdateAt, cutoff))).orderBy(documents.lastSignificantUpdateAt).limit(CRON_BATCH_SIZE));
|
|
233039
|
+
if (idle.length === 0)
|
|
233040
|
+
return;
|
|
233041
|
+
logger3.debug({ count: idle.length }, "Reembed cron: processing idle pending-minor changes");
|
|
233042
|
+
let enqueued = 0;
|
|
233043
|
+
for (const row of idle) {
|
|
233044
|
+
await enqueueReembed([row.id], row.workspaceId ?? undefined);
|
|
233045
|
+
enqueued += 1;
|
|
233046
|
+
}
|
|
233047
|
+
if (enqueued > 0) {
|
|
233048
|
+
logger3.info({ scanned: idle.length, enqueued }, "Reembed cron: enqueued idle pending-minor changes");
|
|
233049
|
+
}
|
|
233050
|
+
}
|
|
233051
|
+
|
|
232732
233052
|
// ../../backend/src/queue/adapters.ts
|
|
232733
233053
|
init_schema2();
|
|
232734
233054
|
init_drizzle_orm();
|
|
@@ -232782,15 +233102,62 @@ async function activateEmbeddingGeneration(documentId, generationId, expectedChu
|
|
|
232782
233102
|
// ../../backend/src/queue/adapters.ts
|
|
232783
233103
|
init_config();
|
|
232784
233104
|
|
|
233105
|
+
// ../../backend/src/lib/graph/generation-state.ts
|
|
233106
|
+
function literal2(value) {
|
|
233107
|
+
return JSON.stringify(value);
|
|
233108
|
+
}
|
|
233109
|
+
function isStaleRevisionError(error53) {
|
|
233110
|
+
return error53 instanceof Error && error53.name === "stale_revision";
|
|
233111
|
+
}
|
|
233112
|
+
async function runGenerationFencedGraphWrite(dependencies) {
|
|
233113
|
+
if (!await dependencies.lockCurrentGeneration()) {
|
|
233114
|
+
const error53 = new Error("graph generation is stale");
|
|
233115
|
+
error53.name = "stale_revision";
|
|
233116
|
+
throw error53;
|
|
233117
|
+
}
|
|
233118
|
+
await dependencies.persist();
|
|
233119
|
+
}
|
|
233120
|
+
function graphReplacementCyphers(identity) {
|
|
233121
|
+
const documentId = literal2(identity.documentId);
|
|
233122
|
+
const generationId = literal2(identity.generationId);
|
|
233123
|
+
const revision = literal2(identity.revision);
|
|
233124
|
+
const timestamp3 = literal2(identity.timestamp);
|
|
233125
|
+
return [
|
|
233126
|
+
`MATCH (d:Document {id: ${documentId}})-[legacy:MENTIONS]->()
|
|
233127
|
+
DELETE legacy RETURN count(legacy)`,
|
|
233128
|
+
`MATCH ()-[r]->() WHERE r.document_id = ${documentId}
|
|
233129
|
+
DELETE r RETURN count(r)`,
|
|
233130
|
+
`MERGE (d:Document {id: ${documentId}})
|
|
233131
|
+
SET d.generation_id = ${generationId}, d.revision = ${revision},
|
|
233132
|
+
d.created_at = coalesce(d.created_at, ${timestamp3}),
|
|
233133
|
+
d.entity_extracted_at = ${timestamp3}
|
|
233134
|
+
RETURN d.id`
|
|
233135
|
+
];
|
|
233136
|
+
}
|
|
233137
|
+
function graphCompensationCyphers(documentIdValue, generationIdValue) {
|
|
233138
|
+
const documentId = literal2(documentIdValue);
|
|
233139
|
+
const generationId = literal2(generationIdValue);
|
|
233140
|
+
return [
|
|
233141
|
+
`MATCH ()-[r]->()
|
|
233142
|
+
WHERE r.document_id = ${documentId}
|
|
233143
|
+
AND r.generation_id = ${generationId}
|
|
233144
|
+
DELETE r RETURN count(r)`,
|
|
233145
|
+
`MATCH (d:Document {id: ${documentId}})
|
|
233146
|
+
WHERE d.generation_id = ${generationId}
|
|
233147
|
+
DETACH DELETE d RETURN count(d)`
|
|
233148
|
+
];
|
|
233149
|
+
}
|
|
233150
|
+
|
|
232785
233151
|
// ../../backend/src/lib/graph/delete-document-state.ts
|
|
232786
|
-
async function
|
|
233152
|
+
async function deleteDocumentGraphGeneration(documentId, generationId) {
|
|
232787
233153
|
const sql6 = await getGraphDb();
|
|
232788
233154
|
if (!sql6)
|
|
232789
233155
|
return;
|
|
232790
|
-
const literal2 = JSON.stringify(documentId);
|
|
232791
233156
|
await sql6.begin(async (tx) => {
|
|
232792
233157
|
await tx.unsafe("SET LOCAL search_path = ag_catalog, public");
|
|
232793
|
-
|
|
233158
|
+
for (const cypher of graphCompensationCyphers(documentId, generationId)) {
|
|
233159
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (deleted agtype)`);
|
|
233160
|
+
}
|
|
232794
233161
|
});
|
|
232795
233162
|
}
|
|
232796
233163
|
|
|
@@ -232813,8 +233180,9 @@ var RELATION_TYPES = [
|
|
|
232813
233180
|
"AUTHORED_BY"
|
|
232814
233181
|
];
|
|
232815
233182
|
var EXTRACT_DEDUP_TTL_SECONDS = 24 * 60 * 60;
|
|
232816
|
-
function extractDedupKey(documentId, chunkIndex, chunkHash2) {
|
|
232817
|
-
|
|
233183
|
+
function extractDedupKey(documentId, chunkIndex, chunkHash2, generationId) {
|
|
233184
|
+
const generation = generationId ? `${generationId}:` : "";
|
|
233185
|
+
return `hiai-docs:extract:done:${documentId}:${generation}${chunkIndex}:${chunkHash2}`;
|
|
232818
233186
|
}
|
|
232819
233187
|
var DEFAULT_MAX_TOKENS = 1024;
|
|
232820
233188
|
var DEFAULT_TEMPERATURE = 0;
|
|
@@ -232835,12 +233203,24 @@ function setCachedEntity(name, type, confidence) {
|
|
|
232835
233203
|
async function extractEntities(chunkText2, documentId, options = {}) {
|
|
232836
233204
|
if (!config3.GRAPH_EXTRACT_ENABLED)
|
|
232837
233205
|
return [];
|
|
233206
|
+
let claimedDedupKey;
|
|
233207
|
+
const releaseDedupClaim = async () => {
|
|
233208
|
+
if (!claimedDedupKey)
|
|
233209
|
+
return;
|
|
233210
|
+
try {
|
|
233211
|
+
await redis.del(claimedDedupKey);
|
|
233212
|
+
} catch (err) {
|
|
233213
|
+
logger3.warn({ err, documentId, claimedDedupKey }, "Failed to release extract-dedup slot");
|
|
233214
|
+
}
|
|
233215
|
+
};
|
|
232838
233216
|
if (options.chunkHash && options.chunkIndex !== undefined) {
|
|
232839
|
-
const key = extractDedupKey(documentId, options.chunkIndex, options.chunkHash);
|
|
233217
|
+
const key = extractDedupKey(documentId, options.chunkIndex, options.chunkHash, options.generationId);
|
|
232840
233218
|
let acquired = false;
|
|
232841
233219
|
try {
|
|
232842
233220
|
const result = await redis.set(key, "1", "EX", EXTRACT_DEDUP_TTL_SECONDS, "NX");
|
|
232843
233221
|
acquired = result === "OK";
|
|
233222
|
+
if (acquired)
|
|
233223
|
+
claimedDedupKey = key;
|
|
232844
233224
|
} catch (err) {
|
|
232845
233225
|
logger3.warn({ err, documentId, chunkIndex: options.chunkIndex }, "Extract-dedup Redis SET failed \u2014 falling through to extraction");
|
|
232846
233226
|
acquired = true;
|
|
@@ -232849,22 +233229,31 @@ async function extractEntities(chunkText2, documentId, options = {}) {
|
|
|
232849
233229
|
return [];
|
|
232850
233230
|
}
|
|
232851
233231
|
const sql6 = await getGraphDb();
|
|
232852
|
-
if (!sql6)
|
|
233232
|
+
if (!sql6) {
|
|
233233
|
+
await releaseDedupClaim();
|
|
232853
233234
|
return [];
|
|
232854
|
-
|
|
233235
|
+
}
|
|
233236
|
+
if (!chunkText2) {
|
|
233237
|
+
await releaseDedupClaim();
|
|
232855
233238
|
return [];
|
|
233239
|
+
}
|
|
232856
233240
|
let entities;
|
|
232857
233241
|
try {
|
|
232858
233242
|
entities = await callEntityExtractionLLM(chunkText2, options);
|
|
232859
233243
|
} catch (err) {
|
|
233244
|
+
await releaseDedupClaim();
|
|
232860
233245
|
logger3.warn({ err, documentId }, "Entity extraction LLM call failed \u2014 skipping");
|
|
232861
233246
|
return [];
|
|
232862
233247
|
}
|
|
232863
|
-
if (entities.length === 0)
|
|
232864
|
-
return [];
|
|
232865
233248
|
try {
|
|
232866
|
-
await persistEntities(sql6, documentId, entities
|
|
233249
|
+
await persistEntities(sql6, documentId, entities, {
|
|
233250
|
+
generationId: options.generationId ?? "legacy",
|
|
233251
|
+
revision: options.revision ?? "legacy"
|
|
233252
|
+
});
|
|
232867
233253
|
} catch (err) {
|
|
233254
|
+
await releaseDedupClaim();
|
|
233255
|
+
if (isStaleRevisionError(err))
|
|
233256
|
+
throw err;
|
|
232868
233257
|
logger3.warn({ err, documentId, count: entities.length }, "Failed to persist extracted entities to AGE \u2014 discarding");
|
|
232869
233258
|
return [];
|
|
232870
233259
|
}
|
|
@@ -233033,30 +233422,55 @@ function stripMarkdownFences(text4) {
|
|
|
233033
233422
|
const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(text4.trim());
|
|
233034
233423
|
return fenced?.[1] ?? text4;
|
|
233035
233424
|
}
|
|
233036
|
-
async function persistEntities(sql6, documentId, entities) {
|
|
233425
|
+
async function persistEntities(sql6, documentId, entities, identity) {
|
|
233037
233426
|
await sql6.begin(async (tx) => {
|
|
233038
233427
|
await tx`SELECT pg_catalog.set_config('search_path', 'ag_catalog, "$user", public', false)`;
|
|
233428
|
+
await tx`SELECT pg_catalog.set_config('app.current_user_id', '00000000-0000-0000-0000-000000000000', true)`;
|
|
233429
|
+
await tx`SELECT pg_catalog.set_config('app.current_user_role', 'admin', true)`;
|
|
233430
|
+
await tx`SELECT pg_catalog.set_config('app.current_workspace_id', '', true)`;
|
|
233039
233431
|
const nowIso = new Date().toISOString();
|
|
233040
|
-
|
|
233041
|
-
|
|
233042
|
-
|
|
233043
|
-
|
|
233044
|
-
|
|
233045
|
-
|
|
233046
|
-
|
|
233047
|
-
|
|
233048
|
-
|
|
233049
|
-
|
|
233050
|
-
|
|
233051
|
-
|
|
233052
|
-
|
|
233053
|
-
|
|
233054
|
-
|
|
233055
|
-
|
|
233056
|
-
|
|
233057
|
-
|
|
233432
|
+
await runGenerationFencedGraphWrite({
|
|
233433
|
+
async lockCurrentGeneration() {
|
|
233434
|
+
const rows = await tx`
|
|
233435
|
+
SELECT d.id
|
|
233436
|
+
FROM public.documents d
|
|
233437
|
+
JOIN public.document_pipeline_runs r
|
|
233438
|
+
ON r.document_id = d.id
|
|
233439
|
+
AND r.generation_id = ${identity.generationId}::uuid
|
|
233440
|
+
WHERE d.id = ${documentId}::uuid
|
|
233441
|
+
AND d.active_embedding_generation = ${identity.generationId}::uuid
|
|
233442
|
+
AND d.content_hash = ${identity.revision}
|
|
233443
|
+
AND r.revision = ${identity.revision}
|
|
233444
|
+
AND r.status NOT IN ('cancelled', 'failed')
|
|
233445
|
+
AND r.graph_status = 'processing'
|
|
233446
|
+
FOR UPDATE OF d, r
|
|
233447
|
+
`;
|
|
233448
|
+
return rows.length === 1;
|
|
233449
|
+
},
|
|
233450
|
+
async persist() {
|
|
233451
|
+
for (const cypher of graphReplacementCyphers({
|
|
233452
|
+
documentId,
|
|
233453
|
+
generationId: identity.generationId,
|
|
233454
|
+
revision: identity.revision,
|
|
233455
|
+
timestamp: nowIso
|
|
233456
|
+
})) {
|
|
233457
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
|
|
233458
|
+
}
|
|
233459
|
+
for (const ent of entities) {
|
|
233460
|
+
const label = ent.type;
|
|
233461
|
+
const name = ent.name;
|
|
233462
|
+
const conf = ent.confidence;
|
|
233463
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${entityUpsertCypher(documentId, label, name, nowIso, conf)} $$) AS (result agtype)`);
|
|
233464
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${documentEntityEdgeCypher(documentId, identity.generationId, label, name, conf)} $$) AS (result agtype)`);
|
|
233465
|
+
}
|
|
233466
|
+
for (const ent of entities) {
|
|
233467
|
+
for (const rel of ent.relationships) {
|
|
233468
|
+
const cypher = entityRelationCypher(documentId, identity.generationId, ent.type, ent.name, rel.targetName, rel.relationType, rel.confidence);
|
|
233469
|
+
await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
|
|
233470
|
+
}
|
|
233471
|
+
}
|
|
233058
233472
|
}
|
|
233059
|
-
}
|
|
233473
|
+
});
|
|
233060
233474
|
});
|
|
233061
233475
|
}
|
|
233062
233476
|
function entityUpsertCypher(docId, label, name, nowIso, confidence) {
|
|
@@ -233065,27 +233479,118 @@ function entityUpsertCypher(docId, label, name, nowIso, confidence) {
|
|
|
233065
233479
|
MERGE (e:\`${label}\` {name: $name})
|
|
233066
233480
|
SET e.created_at = $now, e.last_seen_doc = $docId, e.confidence = $conf
|
|
233067
233481
|
RETURN e.name
|
|
233068
|
-
`.
|
|
233482
|
+
`.replaceAll("$name", JSON.stringify(name)).replaceAll("$now", JSON.stringify(nowIso)).replaceAll("$docId", JSON.stringify(docId)).replaceAll("$conf", confLiteral);
|
|
233069
233483
|
}
|
|
233070
|
-
function documentEntityEdgeCypher(docId, label, name, confidence) {
|
|
233484
|
+
function documentEntityEdgeCypher(docId, generationId, label, name, confidence) {
|
|
233071
233485
|
const confLiteral = typeof confidence === "number" ? JSON.stringify(confidence) : "null";
|
|
233072
233486
|
return `
|
|
233073
233487
|
MATCH (d:Document {id: $docId})
|
|
233074
233488
|
MATCH (e:\`${label}\` {name: $name})
|
|
233075
|
-
MERGE (d)-[r:MENTIONS]->(e)
|
|
233076
|
-
SET r.created_at = $now, r.confidence = $conf
|
|
233489
|
+
MERGE (d)-[r:MENTIONS {document_id: $docId}]->(e)
|
|
233490
|
+
SET r.created_at = $now, r.confidence = $conf, r.generation_id = $generationId
|
|
233077
233491
|
RETURN r
|
|
233078
|
-
`.
|
|
233492
|
+
`.replaceAll("$docId", JSON.stringify(docId)).replaceAll("$name", JSON.stringify(name)).replaceAll("$now", JSON.stringify(new Date().toISOString())).replaceAll("$generationId", JSON.stringify(generationId)).replaceAll("$conf", confLiteral);
|
|
233079
233493
|
}
|
|
233080
|
-
function entityRelationCypher(sourceLabel, sourceName, targetName, relationType, confidence) {
|
|
233494
|
+
function entityRelationCypher(documentId, generationId, sourceLabel, sourceName, targetName, relationType, confidence) {
|
|
233081
233495
|
const confLiteral = typeof confidence === "number" ? JSON.stringify(confidence) : "null";
|
|
233082
233496
|
return `
|
|
233083
233497
|
MATCH (a:\`${sourceLabel}\` {name: $source})
|
|
233084
233498
|
MATCH (b {name: $target})
|
|
233085
|
-
MERGE (a)-[r:\`${relationType}\`]->(b)
|
|
233086
|
-
SET r.created_at = $now, r.confidence = $conf
|
|
233499
|
+
MERGE (a)-[r:\`${relationType}\` {document_id: $documentId}]->(b)
|
|
233500
|
+
SET r.created_at = $now, r.confidence = $conf, r.generation_id = $generationId
|
|
233087
233501
|
RETURN r
|
|
233088
|
-
`.
|
|
233502
|
+
`.replaceAll("$source", JSON.stringify(sourceName)).replaceAll("$target", JSON.stringify(targetName)).replaceAll("$documentId", JSON.stringify(documentId)).replaceAll("$generationId", JSON.stringify(generationId)).replaceAll("$now", JSON.stringify(new Date().toISOString())).replaceAll("$conf", confLiteral);
|
|
233503
|
+
}
|
|
233504
|
+
|
|
233505
|
+
// ../../backend/src/lib/knowledge-summary.ts
|
|
233506
|
+
async function runKnowledgeSummaryStage(dependencies) {
|
|
233507
|
+
const document2 = await dependencies.readCurrent();
|
|
233508
|
+
if (!document2)
|
|
233509
|
+
return "cancelled";
|
|
233510
|
+
const summary = await dependencies.generate(document2);
|
|
233511
|
+
if (summary.status === "skipped")
|
|
233512
|
+
return "skipped";
|
|
233513
|
+
return await dependencies.persistIfCurrent(summary) ? "ready" : "cancelled";
|
|
233514
|
+
}
|
|
233515
|
+
async function buildKnowledgeSummary(document2, provider) {
|
|
233516
|
+
if (!document2.content.trim()) {
|
|
233517
|
+
return { status: "skipped", reason: "empty_document" };
|
|
233518
|
+
}
|
|
233519
|
+
const output = await provider(document2);
|
|
233520
|
+
if (!output)
|
|
233521
|
+
throw new Error("summary_provider_failed");
|
|
233522
|
+
const language = output.language.trim();
|
|
233523
|
+
const description = output.description.trim();
|
|
233524
|
+
if (!language || !description)
|
|
233525
|
+
throw new Error("summary_provider_invalid");
|
|
233526
|
+
const seen = new Set;
|
|
233527
|
+
const keywords = [];
|
|
233528
|
+
for (const raw of output.keywords) {
|
|
233529
|
+
const keyword = raw.trim();
|
|
233530
|
+
const key = keyword.toLocaleLowerCase("en");
|
|
233531
|
+
if (!keyword || seen.has(key))
|
|
233532
|
+
continue;
|
|
233533
|
+
seen.add(key);
|
|
233534
|
+
keywords.push(keyword);
|
|
233535
|
+
if (keywords.length === 20)
|
|
233536
|
+
break;
|
|
233537
|
+
}
|
|
233538
|
+
return {
|
|
233539
|
+
status: "ready",
|
|
233540
|
+
language,
|
|
233541
|
+
description,
|
|
233542
|
+
keywords,
|
|
233543
|
+
model: output.model
|
|
233544
|
+
};
|
|
233545
|
+
}
|
|
233546
|
+
|
|
233547
|
+
// ../../backend/src/lib/knowledge-summary-provider.ts
|
|
233548
|
+
init_zod();
|
|
233549
|
+
init_config();
|
|
233550
|
+
var summarySchema = exports_external2.object({
|
|
233551
|
+
language: exports_external2.string().min(1).max(32),
|
|
233552
|
+
description: exports_external2.string().min(1).max(2000),
|
|
233553
|
+
keywords: exports_external2.array(exports_external2.string().min(1).max(100)).max(20)
|
|
233554
|
+
});
|
|
233555
|
+
var systemPrompt = [
|
|
233556
|
+
"Summarize a knowledge-base document for retrieval metadata.",
|
|
233557
|
+
"Return only JSON with language, description, and keywords.",
|
|
233558
|
+
"Use a concise factual description and at most 20 distinct keywords."
|
|
233559
|
+
].join(`
|
|
233560
|
+
`);
|
|
233561
|
+
function provider(baseUrl, apiKey, model) {
|
|
233562
|
+
if (!baseUrl || !model)
|
|
233563
|
+
return;
|
|
233564
|
+
return {
|
|
233565
|
+
baseUrl,
|
|
233566
|
+
model,
|
|
233567
|
+
apiKey: resolveChatProviderKey(baseUrl, apiKey, config3.OPENROUTER_API_KEY),
|
|
233568
|
+
timeoutMs: config3.GRAPH_EXTRACT_TIMEOUT_MS,
|
|
233569
|
+
reasoningEffort: config3.GRAPH_EXTRACT_REASONING_EFFORT
|
|
233570
|
+
};
|
|
233571
|
+
}
|
|
233572
|
+
async function requestKnowledgeSummary(document2) {
|
|
233573
|
+
const primary = provider(config3.GRAPH_EXTRACT_BASE_URL, config3.GRAPH_EXTRACT_API_KEY, config3.GRAPH_EXTRACT_MODEL);
|
|
233574
|
+
if (!primary)
|
|
233575
|
+
return null;
|
|
233576
|
+
const fallback = provider(config3.GRAPH_EXTRACT_FALLBACK_BASE_URL, config3.GRAPH_EXTRACT_FALLBACK_API_KEY, config3.GRAPH_EXTRACT_FALLBACK_MODEL);
|
|
233577
|
+
const result = await requestStructuredChat({
|
|
233578
|
+
primary,
|
|
233579
|
+
fallback,
|
|
233580
|
+
messages: [
|
|
233581
|
+
{ role: "system", content: systemPrompt },
|
|
233582
|
+
{
|
|
233583
|
+
role: "user",
|
|
233584
|
+
content: `Title: ${document2.title}
|
|
233585
|
+
|
|
233586
|
+
${document2.content.slice(0, 50000)}`
|
|
233587
|
+
}
|
|
233588
|
+
],
|
|
233589
|
+
outputSchema: summarySchema,
|
|
233590
|
+
maxTokens: 768,
|
|
233591
|
+
temperature: 0
|
|
233592
|
+
});
|
|
233593
|
+
return result ? { ...result.data, model: result.model } : null;
|
|
233089
233594
|
}
|
|
233090
233595
|
|
|
233091
233596
|
// ../../backend/src/queue/document-revision.ts
|
|
@@ -233263,6 +233768,23 @@ async function getRun(generationId) {
|
|
|
233263
233768
|
return run4 ?? null;
|
|
233264
233769
|
});
|
|
233265
233770
|
}
|
|
233771
|
+
async function isCurrentPipelineGeneration(job2) {
|
|
233772
|
+
return withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
|
|
233773
|
+
const [current] = await tx.select({ id: documents.id }).from(documents).innerJoin(documentPipelineRuns, and(eq(documentPipelineRuns.documentId, documents.id), eq(documentPipelineRuns.generationId, job2.generationId))).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documentPipelineRuns.revision, job2.revision), ne(documentPipelineRuns.status, "cancelled"), ne(documentPipelineRuns.status, "failed"), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
233774
|
+
return Boolean(current);
|
|
233775
|
+
});
|
|
233776
|
+
}
|
|
233777
|
+
async function cancelStalePipelineRun(job2) {
|
|
233778
|
+
await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
|
|
233779
|
+
status: "cancelled",
|
|
233780
|
+
graphStatus: "cancelled",
|
|
233781
|
+
summarizeStatus: "cancelled",
|
|
233782
|
+
finalizeStatus: "cancelled",
|
|
233783
|
+
errorCode: "stale_revision",
|
|
233784
|
+
completedAt: new Date,
|
|
233785
|
+
updatedAt: new Date
|
|
233786
|
+
}).where(and(eq(documentPipelineRuns.generationId, job2.generationId), ne(documentPipelineRuns.status, "cancelled"))));
|
|
233787
|
+
}
|
|
233266
233788
|
async function setStageStatus(generationId, stage, status2, errorCode) {
|
|
233267
233789
|
await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
|
|
233268
233790
|
...stagePatch(stage, status2),
|
|
@@ -233518,18 +234040,22 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
233518
234040
|
},
|
|
233519
234041
|
async extract(job2) {
|
|
233520
234042
|
const doc5 = await withTenant(admin, async (tx) => {
|
|
233521
|
-
const [row] = await tx.select({ content: documents.content }).from(documents).where(and(eq(documents.id, job2.documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
234043
|
+
const [row] = await tx.select({ content: documents.content }).from(documents).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documents.contentHash, job2.revision), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
233522
234044
|
return row;
|
|
233523
234045
|
});
|
|
233524
234046
|
if (!doc5)
|
|
233525
234047
|
throw new Error("Pipeline document not found");
|
|
233526
|
-
await withProviderPermit(providerProfile(`graph:${config3.GRAPH_EXTRACT_MODEL ?? "default"}`), "graph", () => extractEntities(doc5.content ?? "", job2.documentId
|
|
234048
|
+
await withProviderPermit(providerProfile(`graph:${config3.GRAPH_EXTRACT_MODEL ?? "default"}`), "graph", () => extractEntities(doc5.content ?? "", job2.documentId, {
|
|
234049
|
+
generationId: job2.generationId,
|
|
234050
|
+
revision: job2.revision
|
|
234051
|
+
}));
|
|
233527
234052
|
},
|
|
233528
234053
|
async compensateExtract(job2) {
|
|
233529
234054
|
const owned = await withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => tx.select({ id: documents.id }).from(documents).where(and(eq(documents.id, job2.documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1));
|
|
233530
234055
|
if (owned.length === 1)
|
|
233531
|
-
await
|
|
234056
|
+
await deleteDocumentGraphGeneration(job2.documentId, job2.generationId);
|
|
233532
234057
|
},
|
|
234058
|
+
cancelStaleRun: cancelStalePipelineRun,
|
|
233533
234059
|
setGraphStatus: (generationId, status2, errorCode) => setStageStatus(generationId, "graph", status2, errorCode),
|
|
233534
234060
|
async enqueueSummarize(job2) {
|
|
233535
234061
|
const data = { ...job2, stage: "summarize" };
|
|
@@ -233542,6 +234068,8 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
233542
234068
|
},
|
|
233543
234069
|
summarize: {
|
|
233544
234070
|
isCancelled: async (job2) => (await getRun(job2.generationId))?.status === "cancelled",
|
|
234071
|
+
isCurrent: isCurrentPipelineGeneration,
|
|
234072
|
+
cancelStaleRun: cancelStalePipelineRun,
|
|
233545
234073
|
async getRun(job2) {
|
|
233546
234074
|
const run4 = await getRun(job2.generationId);
|
|
233547
234075
|
return run4 ? {
|
|
@@ -233552,8 +234080,59 @@ function createPipelineStageDependencies(redisUrl) {
|
|
|
233552
234080
|
embedStatus: pipelineStageStatus(run4.embedStatus)
|
|
233553
234081
|
} : null;
|
|
233554
234082
|
},
|
|
233555
|
-
enabled: () =>
|
|
233556
|
-
async summarize() {
|
|
234083
|
+
enabled: () => config3.GRAPH_EXTRACT_ENABLED && Boolean(config3.GRAPH_EXTRACT_BASE_URL && config3.GRAPH_EXTRACT_MODEL),
|
|
234084
|
+
async summarize(job2) {
|
|
234085
|
+
return runKnowledgeSummaryStage({
|
|
234086
|
+
readCurrent: () => withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
|
|
234087
|
+
const [document2] = await tx.select({
|
|
234088
|
+
title: documents.title,
|
|
234089
|
+
content: documents.content,
|
|
234090
|
+
contentHash: documents.contentHash,
|
|
234091
|
+
activeGeneration: documents.activeEmbeddingGeneration
|
|
234092
|
+
}).from(documents).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
|
|
234093
|
+
if (!document2)
|
|
234094
|
+
return null;
|
|
234095
|
+
const revision = resolveDocumentRevision(document2.contentHash, document2.title, document2.content ?? "");
|
|
234096
|
+
if (revision !== job2.revision)
|
|
234097
|
+
return null;
|
|
234098
|
+
return {
|
|
234099
|
+
title: document2.title,
|
|
234100
|
+
content: document2.content ?? "",
|
|
234101
|
+
revision
|
|
234102
|
+
};
|
|
234103
|
+
}),
|
|
234104
|
+
generate: (document2) => buildKnowledgeSummary(document2, requestKnowledgeSummary),
|
|
234105
|
+
persistIfCurrent: (summary) => withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
|
|
234106
|
+
const [current] = await tx.select({ id: documents.id }).from(documents).innerJoin(documentPipelineRuns, and(eq(documentPipelineRuns.documentId, documents.id), eq(documentPipelineRuns.generationId, job2.generationId))).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documents.contentHash, job2.revision), eq(documentPipelineRuns.revision, job2.revision), eq(documentPipelineRuns.summarizeStatus, "processing"), ne(documentPipelineRuns.status, "cancelled"), ne(documentPipelineRuns.status, "failed"), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1).for("update");
|
|
234107
|
+
if (!current)
|
|
234108
|
+
return false;
|
|
234109
|
+
await tx.insert(documentKnowledgeSummaries).values({
|
|
234110
|
+
document_id: job2.documentId,
|
|
234111
|
+
owner_id: job2.ownerId,
|
|
234112
|
+
workspace_id: job2.workspaceId,
|
|
234113
|
+
generation_id: job2.generationId,
|
|
234114
|
+
revision: job2.revision,
|
|
234115
|
+
language: summary.language,
|
|
234116
|
+
description: summary.description,
|
|
234117
|
+
keywords: summary.keywords,
|
|
234118
|
+
updated_at: new Date
|
|
234119
|
+
}).onConflictDoUpdate({
|
|
234120
|
+
target: [
|
|
234121
|
+
documentKnowledgeSummaries.document_id,
|
|
234122
|
+
documentKnowledgeSummaries.generation_id
|
|
234123
|
+
],
|
|
234124
|
+
set: {
|
|
234125
|
+
revision: job2.revision,
|
|
234126
|
+
language: summary.language,
|
|
234127
|
+
description: summary.description,
|
|
234128
|
+
keywords: summary.keywords,
|
|
234129
|
+
updated_at: new Date
|
|
234130
|
+
}
|
|
234131
|
+
});
|
|
234132
|
+
return true;
|
|
234133
|
+
})
|
|
234134
|
+
});
|
|
234135
|
+
},
|
|
233557
234136
|
setSummaryStatus: (generationId, status2, errorCode) => setStageStatus(generationId, "summarize", status2, errorCode),
|
|
233558
234137
|
async enqueueFinalize(job2) {
|
|
233559
234138
|
const data = { ...job2, stage: "finalize" };
|
|
@@ -233938,10 +234517,8 @@ function createEmbedWorker(redisUrl, deps, options = {}) {
|
|
|
233938
234517
|
});
|
|
233939
234518
|
}
|
|
233940
234519
|
|
|
233941
|
-
// ../../backend/src/queue/
|
|
234520
|
+
// ../../backend/src/queue/stage-policies.ts
|
|
233942
234521
|
function deriveFinalStatus(run4) {
|
|
233943
|
-
if (!run4)
|
|
233944
|
-
throw new Error("Pipeline run not found");
|
|
233945
234522
|
if (run4.status === "cancelled")
|
|
233946
234523
|
return "cancelled";
|
|
233947
234524
|
if (run4.embedStatus === "failed" || run4.embedStatus === "cancelled")
|
|
@@ -233956,6 +234533,96 @@ function deriveFinalStatus(run4) {
|
|
|
233956
234533
|
}
|
|
233957
234534
|
return "failed";
|
|
233958
234535
|
}
|
|
234536
|
+
async function processSummaryStage(job2, dependencies) {
|
|
234537
|
+
const isExplicitlyCancelled = async () => await dependencies.isCancelled?.(job2) === true;
|
|
234538
|
+
const hasCurrentGeneration = async () => await dependencies.isCurrent?.(job2) !== false;
|
|
234539
|
+
const continueIfCurrent = async () => {
|
|
234540
|
+
if (await isExplicitlyCancelled())
|
|
234541
|
+
return false;
|
|
234542
|
+
if (await hasCurrentGeneration())
|
|
234543
|
+
return true;
|
|
234544
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234545
|
+
return false;
|
|
234546
|
+
};
|
|
234547
|
+
if (!await continueIfCurrent())
|
|
234548
|
+
return;
|
|
234549
|
+
const run4 = await dependencies.getRun(job2);
|
|
234550
|
+
if (!run4)
|
|
234551
|
+
throw new Error("Pipeline run not found");
|
|
234552
|
+
if (run4.ownerId !== job2.ownerId || run4.documentId !== job2.documentId) {
|
|
234553
|
+
throw new Error("Pipeline owner mismatch");
|
|
234554
|
+
}
|
|
234555
|
+
if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
|
|
234556
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234557
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234558
|
+
return;
|
|
234559
|
+
}
|
|
234560
|
+
if (!dependencies.enabled()) {
|
|
234561
|
+
if (!await continueIfCurrent())
|
|
234562
|
+
return;
|
|
234563
|
+
await dependencies.setSummaryStatus(job2.generationId, "skipped");
|
|
234564
|
+
if (!await continueIfCurrent())
|
|
234565
|
+
return;
|
|
234566
|
+
await dependencies.enqueueFinalize(job2);
|
|
234567
|
+
return;
|
|
234568
|
+
}
|
|
234569
|
+
if (!await continueIfCurrent())
|
|
234570
|
+
return;
|
|
234571
|
+
await dependencies.setSummaryStatus(job2.generationId, "processing");
|
|
234572
|
+
try {
|
|
234573
|
+
if (!await continueIfCurrent())
|
|
234574
|
+
return;
|
|
234575
|
+
const status2 = await dependencies.summarize(job2);
|
|
234576
|
+
if (status2 === "cancelled") {
|
|
234577
|
+
if (await isExplicitlyCancelled()) {
|
|
234578
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled");
|
|
234579
|
+
return;
|
|
234580
|
+
}
|
|
234581
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234582
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234583
|
+
return;
|
|
234584
|
+
}
|
|
234585
|
+
if (await isExplicitlyCancelled()) {
|
|
234586
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled");
|
|
234587
|
+
return;
|
|
234588
|
+
}
|
|
234589
|
+
if (!await hasCurrentGeneration()) {
|
|
234590
|
+
if (await isExplicitlyCancelled())
|
|
234591
|
+
return;
|
|
234592
|
+
await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234593
|
+
await dependencies.cancelStaleRun?.(job2);
|
|
234594
|
+
return;
|
|
234595
|
+
}
|
|
234596
|
+
await dependencies.setSummaryStatus(job2.generationId, status2);
|
|
234597
|
+
} catch (error53) {
|
|
234598
|
+
if (!await continueIfCurrent())
|
|
234599
|
+
return;
|
|
234600
|
+
await dependencies.setSummaryStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "summary_failed");
|
|
234601
|
+
if (!await continueIfCurrent())
|
|
234602
|
+
return;
|
|
234603
|
+
await dependencies.enqueueFinalize(job2);
|
|
234604
|
+
return;
|
|
234605
|
+
}
|
|
234606
|
+
if (!await continueIfCurrent())
|
|
234607
|
+
return;
|
|
234608
|
+
await dependencies.enqueueFinalize(job2);
|
|
234609
|
+
}
|
|
234610
|
+
async function processGraphStageFailure(job2, error53, dependencies) {
|
|
234611
|
+
if (isStaleRevisionError(error53)) {
|
|
234612
|
+
await dependencies.setGraphStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234613
|
+
await dependencies.cancelStaleRun(job2);
|
|
234614
|
+
return;
|
|
234615
|
+
}
|
|
234616
|
+
if (await dependencies.isCancelled?.(job2))
|
|
234617
|
+
throw error53;
|
|
234618
|
+
await dependencies.setGraphStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "graph_failed");
|
|
234619
|
+
if (!await dependencies.isCancelled?.(job2)) {
|
|
234620
|
+
await dependencies.enqueueSummarize(job2);
|
|
234621
|
+
}
|
|
234622
|
+
throw error53;
|
|
234623
|
+
}
|
|
234624
|
+
|
|
234625
|
+
// ../../backend/src/queue/workers/finalize.worker.ts
|
|
233959
234626
|
function createFinalizeWorker(deps) {
|
|
233960
234627
|
return async function processFinalizeJob(input) {
|
|
233961
234628
|
const job2 = finalizeJobSchema.parse(input);
|
|
@@ -233987,6 +234654,7 @@ function createGraphWorker(deps) {
|
|
|
233987
234654
|
}
|
|
233988
234655
|
if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
|
|
233989
234656
|
await deps.setGraphStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234657
|
+
await deps.cancelStaleRun(job2);
|
|
233990
234658
|
return;
|
|
233991
234659
|
}
|
|
233992
234660
|
if (run4.embedStatus !== "ready") {
|
|
@@ -234012,12 +234680,7 @@ function createGraphWorker(deps) {
|
|
|
234012
234680
|
if (!await deps.isCancelled?.(job2))
|
|
234013
234681
|
await deps.enqueueSummarize(job2);
|
|
234014
234682
|
} catch (error53) {
|
|
234015
|
-
|
|
234016
|
-
throw error53;
|
|
234017
|
-
await deps.setGraphStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "graph_failed");
|
|
234018
|
-
if (!await deps.isCancelled?.(job2))
|
|
234019
|
-
await deps.enqueueSummarize(job2);
|
|
234020
|
-
throw error53;
|
|
234683
|
+
await processGraphStageFailure(job2, error53, deps);
|
|
234021
234684
|
}
|
|
234022
234685
|
};
|
|
234023
234686
|
}
|
|
@@ -234088,44 +234751,7 @@ function createPrepareWorker(redisUrl, deps, options = {}) {
|
|
|
234088
234751
|
function createSummarizeWorker(deps) {
|
|
234089
234752
|
return async function processSummarizeJob(input) {
|
|
234090
234753
|
const job2 = summarizeJobSchema.parse(input);
|
|
234091
|
-
|
|
234092
|
-
return;
|
|
234093
|
-
const run4 = await deps.getRun(job2);
|
|
234094
|
-
if (!run4)
|
|
234095
|
-
throw new Error("Pipeline run not found");
|
|
234096
|
-
if (run4.ownerId !== job2.ownerId || run4.documentId !== job2.documentId) {
|
|
234097
|
-
throw new Error("Pipeline owner mismatch");
|
|
234098
|
-
}
|
|
234099
|
-
if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
|
|
234100
|
-
await deps.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
|
|
234101
|
-
return;
|
|
234102
|
-
}
|
|
234103
|
-
if (!deps.enabled()) {
|
|
234104
|
-
if (await deps.isCancelled?.(job2))
|
|
234105
|
-
return;
|
|
234106
|
-
await deps.setSummaryStatus(job2.generationId, "skipped");
|
|
234107
|
-
if (!await deps.isCancelled?.(job2))
|
|
234108
|
-
await deps.enqueueFinalize(job2);
|
|
234109
|
-
return;
|
|
234110
|
-
}
|
|
234111
|
-
if (await deps.isCancelled?.(job2))
|
|
234112
|
-
return;
|
|
234113
|
-
await deps.setSummaryStatus(job2.generationId, "processing");
|
|
234114
|
-
try {
|
|
234115
|
-
if (await deps.isCancelled?.(job2))
|
|
234116
|
-
return;
|
|
234117
|
-
await deps.summarize(job2);
|
|
234118
|
-
if (await deps.isCancelled?.(job2))
|
|
234119
|
-
return;
|
|
234120
|
-
await deps.setSummaryStatus(job2.generationId, "ready");
|
|
234121
|
-
} catch (error53) {
|
|
234122
|
-
if (await deps.isCancelled?.(job2))
|
|
234123
|
-
return;
|
|
234124
|
-
await deps.setSummaryStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "summary_failed");
|
|
234125
|
-
throw error53;
|
|
234126
|
-
}
|
|
234127
|
-
if (!await deps.isCancelled?.(job2))
|
|
234128
|
-
await deps.enqueueFinalize(job2);
|
|
234754
|
+
await processSummaryStage(job2, deps);
|
|
234129
234755
|
};
|
|
234130
234756
|
}
|
|
234131
234757
|
|
|
@@ -234265,6 +234891,7 @@ var pipelineRuntime = await startRegisteredPipelineWorkers({
|
|
|
234265
234891
|
logger3.info({ legacy, recovery }, "Pipeline recovery completed");
|
|
234266
234892
|
}
|
|
234267
234893
|
});
|
|
234894
|
+
var reembedCronRuntime = startReembedCron();
|
|
234268
234895
|
ensureBucket(storage, BUCKET).catch((err) => {
|
|
234269
234896
|
logger3.error({ err }, "Failed to ensure storage bucket");
|
|
234270
234897
|
});
|
|
@@ -234319,7 +234946,7 @@ var swaggerConfig = {
|
|
|
234319
234946
|
},
|
|
234320
234947
|
info: {
|
|
234321
234948
|
title: "DocsMint API",
|
|
234322
|
-
version: "0.6.
|
|
234949
|
+
version: "0.6.2",
|
|
234323
234950
|
description: "Self-hosted AI-first documentation platform. Full-text + semantic search, version history, sharing, and folder organization.",
|
|
234324
234951
|
contact: { name: "HiAi-gg", url: "https://github.com/HiAi-gg/docsmint" },
|
|
234325
234952
|
license: {
|
|
@@ -234390,6 +235017,7 @@ app.listen({
|
|
|
234390
235017
|
logger3.info({ port: config3.API_PORT }, "hiai-docs API started");
|
|
234391
235018
|
var stopDocsMintApi = async () => {
|
|
234392
235019
|
logger3.info("Shutting down...");
|
|
235020
|
+
reembedCronRuntime.close();
|
|
234393
235021
|
await pipelineRuntime.close();
|
|
234394
235022
|
await app.stop();
|
|
234395
235023
|
};
|