@hiai-gg/docsmint 0.6.0 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -42744,6 +42744,7 @@ __export(exports_schema, {
42744
42744
  documentRelations: () => documentRelations,
42745
42745
  documentPipelineRuns: () => documentPipelineRuns,
42746
42746
  documentPipelineBatches: () => documentPipelineBatches,
42747
+ documentKnowledgeSummaries: () => documentKnowledgeSummaries,
42747
42748
  documentEmbeddings: () => documentEmbeddings,
42748
42749
  documentEmbeddingRelations: () => documentEmbeddingRelations,
42749
42750
  documentCreateOperations: () => documentCreateOperations,
@@ -42756,7 +42757,7 @@ __export(exports_schema, {
42756
42757
  apiKeyRelations: () => apiKeyRelations,
42757
42758
  accounts: () => accounts
42758
42759
  });
42759
- var vector4, tsvector, documentVisibilityEnum, shareRoleEnum, shareAccessModeEnum, shareGrantStatusEnum, embeddingStatusEnum, pipelineStageEnum, pipelineStatusEnum, lifecycleOperationKindEnum, lifecycleOperationStatusEnum, users, sessions, accounts, verifications, folders, folderRelations, documents, documentRelations, documentPipelineRuns, documentPipelineBatches, tags, tagRelations, categories, categoryRelations, documentTags, documentTagRelations, shareLinks, shareLinkRelations, guestAccess, guestAccessRelations, attachments, attachmentRelations, versions, documentEmbeddings, documentEmbeddingRelations, apiKeys, apiKeyRelations, auditLog, lifecycleOperations, documentCreateOperations, versionRelations;
42760
+ var vector4, tsvector, documentVisibilityEnum, shareRoleEnum, shareAccessModeEnum, shareGrantStatusEnum, embeddingStatusEnum, pipelineStageEnum, pipelineStatusEnum, lifecycleOperationKindEnum, lifecycleOperationStatusEnum, users, sessions, accounts, verifications, folders, folderRelations, documents, documentRelations, documentPipelineRuns, documentPipelineBatches, documentKnowledgeSummaries, tags, tagRelations, categories, categoryRelations, documentTags, documentTagRelations, shareLinks, shareLinkRelations, guestAccess, guestAccessRelations, attachments, attachmentRelations, versions, documentEmbeddings, documentEmbeddingRelations, apiKeys, apiKeyRelations, auditLog, lifecycleOperations, documentCreateOperations, versionRelations;
42760
42761
  var init_schema2 = __esm(() => {
42761
42762
  init_pg_core();
42762
42763
  init_drizzle_orm();
@@ -43010,6 +43011,22 @@ var init_schema2 = __esm(() => {
43010
43011
  index("document_pipeline_batches_stage_status_available_idx").on(table3.stage, table3.status, table3.availableAt),
43011
43012
  index("document_pipeline_batches_document_id_idx").on(table3.documentId)
43012
43013
  ]);
43014
+ documentKnowledgeSummaries = pgTable("document_knowledge_summaries", {
43015
+ id: uuid().primaryKey().defaultRandom(),
43016
+ document_id: uuid().notNull().references(() => documents.id, { onDelete: "cascade" }),
43017
+ owner_id: uuid().notNull().references(() => users.id, { onDelete: "cascade" }),
43018
+ workspace_id: text(),
43019
+ generation_id: uuid().notNull(),
43020
+ revision: text().notNull(),
43021
+ language: text().notNull(),
43022
+ description: text().notNull(),
43023
+ keywords: text().array().notNull().default(sql`ARRAY[]::text[]`),
43024
+ created_at: timestamp().defaultNow().notNull(),
43025
+ updated_at: timestamp().defaultNow().notNull()
43026
+ }, (table3) => [
43027
+ uniqueIndex("document_knowledge_summaries_document_generation_idx").on(table3.document_id, table3.generation_id),
43028
+ index("document_knowledge_summaries_owner_document_idx").on(table3.owner_id, table3.document_id)
43029
+ ]);
43013
43030
  tags = pgTable("tags", {
43014
43031
  id: uuid("id").primaryKey().defaultRandom(),
43015
43032
  ownerId: uuid("owner_id").notNull().references(() => users.id, { onDelete: "cascade" }),
@@ -173203,6 +173220,8 @@ async function validateApiKey(key) {
173203
173220
  scopes
173204
173221
  };
173205
173222
  }
173223
+ // ../db/src/document-writer.ts
173224
+ init_schema2();
173206
173225
 
173207
173226
  // ../db/src/index.ts
173208
173227
  init_schema2();
@@ -194094,6 +194113,7 @@ function contentAccessForExternalContext(ctx) {
194094
194113
  permissions.add("write");
194095
194114
  } else if (ctx.actorRole === "editor") {
194096
194115
  permissions.add("edit");
194116
+ permissions.add("write");
194097
194117
  }
194098
194118
  return {
194099
194119
  principal: { kind: "session", userId: ctx.userId },
@@ -194144,6 +194164,9 @@ function canAccessContent(access, action) {
194144
194164
  return access.permissions.has(action);
194145
194165
  return !access.restricted || access.permissions.has(action);
194146
194166
  }
194167
+ function canManageCategories(access) {
194168
+ return !access.restricted && canAccessContent(access, "write");
194169
+ }
194147
194170
  function effectiveDocumentCategory(row) {
194148
194171
  return row.categoryId ?? row.folderCategoryId ?? null;
194149
194172
  }
@@ -194237,7 +194260,8 @@ var enqueueDocumentPipelineSchema = exports_external2.object({
194237
194260
  workspaceId: exports_external2.string().min(1).optional(),
194238
194261
  revision: exports_external2.string().min(1),
194239
194262
  source: pipelineSourceSchema,
194240
- requestedAt: exports_external2.iso.datetime().optional()
194263
+ requestedAt: exports_external2.iso.datetime().optional(),
194264
+ forceNewGeneration: exports_external2.boolean().optional()
194241
194265
  });
194242
194266
  var JOB_IDS = {
194243
194267
  prepare: (documentId, generationId, workspaceId) => `prepare-${documentId}-${generationId}${workspaceId ? `-${workspaceId}` : ""}`,
@@ -194266,7 +194290,14 @@ var postgresRunStore = {
194266
194290
  const [document2] = await tx.select({ id: documents.id }).from(documents).where(and(eq(documents.id, input.documentId), ownerBoundary)).limit(1);
194267
194291
  if (!document2)
194268
194292
  throw new Error("Document not found for pipeline owner");
194269
- const [existing] = await tx.select({ generationId: documentPipelineRuns.generationId }).from(documentPipelineRuns).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, eq(documentPipelineRuns.revision, input.revision), inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES]))).limit(1);
194293
+ if (input.forceNewGeneration) {
194294
+ await tx.update(documentPipelineRuns).set({
194295
+ status: "cancelled",
194296
+ errorCode: "superseded_by_reindex",
194297
+ updatedAt: new Date
194298
+ }).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES])));
194299
+ }
194300
+ const [existing] = input.forceNewGeneration ? [] : await tx.select({ generationId: documentPipelineRuns.generationId }).from(documentPipelineRuns).where(and(eq(documentPipelineRuns.documentId, input.documentId), runBoundary, eq(documentPipelineRuns.revision, input.revision), inArray(documentPipelineRuns.status, [...ACTIVE_STATUSES]))).limit(1);
194270
194301
  if (existing)
194271
194302
  return { run: existing, created: false };
194272
194303
  const [created] = await tx.insert(documentPipelineRuns).values({
@@ -194302,7 +194333,8 @@ async function enqueueDocumentPipeline(input, dependencies) {
194302
194333
  const { run, created } = await deps.runs.findOrCreate({
194303
194334
  ...parsed,
194304
194335
  requestedAt,
194305
- generationId: proposedGenerationId
194336
+ generationId: proposedGenerationId,
194337
+ forceNewGeneration: parsed.forceNewGeneration
194306
194338
  });
194307
194339
  if (created) {
194308
194340
  const job2 = {
@@ -194373,10 +194405,10 @@ ${content52}`;
194373
194405
  init_logger3();
194374
194406
  init_redis();
194375
194407
  var LEGACY_EMBEDDING_QUEUE_KEY = "hiai-docs:embedding-queue";
194376
- async function enqueueEmbedding(documentId, source = "interactive") {
194408
+ async function enqueueEmbedding(documentId, source = "interactive", workspaceId, options = {}) {
194377
194409
  let pipelineInput;
194378
194410
  try {
194379
- await withTenant(adminTenantContext(ZERO_UUID), async (tx) => {
194411
+ await withTenant({ ...adminTenantContext(ZERO_UUID), workspaceId }, async (tx) => {
194380
194412
  const rows = await tx.select({
194381
194413
  active: documents.activeEmbeddingGeneration,
194382
194414
  ownerId: documents.ownerId,
@@ -194387,7 +194419,7 @@ async function enqueueEmbedding(documentId, source = "interactive") {
194387
194419
  if (!document2)
194388
194420
  return;
194389
194421
  const revision = contentHash(document2.title, document2.content ?? "");
194390
- pipelineInput = { ownerId: document2.ownerId, revision };
194422
+ pipelineInput = { ownerId: document2.ownerId, revision, workspaceId };
194391
194423
  const activeGeneration = document2.active;
194392
194424
  if (activeGeneration) {
194393
194425
  await tx.update(documents).set({
@@ -194410,8 +194442,10 @@ async function enqueueEmbedding(documentId, source = "interactive") {
194410
194442
  await enqueueDocumentPipeline({
194411
194443
  documentId,
194412
194444
  ownerId: pipelineInput.ownerId,
194445
+ workspaceId: pipelineInput.workspaceId,
194413
194446
  revision: pipelineInput.revision,
194414
- source
194447
+ source,
194448
+ forceNewGeneration: options.forceNewGeneration
194415
194449
  });
194416
194450
  return true;
194417
194451
  } catch (err) {
@@ -194449,7 +194483,7 @@ async function enqueueReembed(docIds, workspaceId) {
194449
194483
  let pushed = 0;
194450
194484
  for (const id2 of unique) {
194451
194485
  if (await claimEnqueueSlot(id2, workspaceId)) {
194452
- enqueueEmbedding(id2);
194486
+ enqueueEmbedding(id2, "interactive", workspaceId);
194453
194487
  pushed += 1;
194454
194488
  }
194455
194489
  }
@@ -196359,9 +196393,6 @@ init_schema2();
196359
196393
  init_drizzle_orm();
196360
196394
  init_zod();
196361
196395
  init_logger3();
196362
- function isCategoryManager(principal) {
196363
- return principal?.kind === "session" || principal?.kind === "operator";
196364
- }
196365
196396
  var createCategorySchema = exports_external2.object({
196366
196397
  name: exports_external2.string().trim().min(1).max(255),
196367
196398
  apiMode: exports_external2.enum(["unavailable", "global", "category"]).optional(),
@@ -196499,9 +196530,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
196499
196530
  set2.status = 401;
196500
196531
  return { error: "Unauthorized" };
196501
196532
  }
196502
- if (!isCategoryManager(access.principal)) {
196533
+ if (!canManageCategories(access)) {
196503
196534
  set2.status = 403;
196504
- return { error: "Browser session or operator credential required" };
196535
+ return { error: "Full workspace write access required" };
196505
196536
  }
196506
196537
  const userId = ctx.userId;
196507
196538
  const parsed = createCategorySchema.safeParse(await request.json());
@@ -196551,9 +196582,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
196551
196582
  set2.status = 401;
196552
196583
  return { error: "Unauthorized" };
196553
196584
  }
196554
- if (!isCategoryManager(access.principal)) {
196585
+ if (!canManageCategories(access)) {
196555
196586
  set2.status = 403;
196556
- return { error: "Browser session or operator credential required" };
196587
+ return { error: "Full workspace write access required" };
196557
196588
  }
196558
196589
  const userId = ctx.userId;
196559
196590
  const parsed = updateCategorySchema.safeParse(await request.json());
@@ -196622,9 +196653,9 @@ var categoryRoutes = new Elysia({ prefix: "/api" }).get("/categories", async ({
196622
196653
  set2.status = 401;
196623
196654
  return { error: "Unauthorized" };
196624
196655
  }
196625
- if (!isCategoryManager(access.principal)) {
196656
+ if (!canManageCategories(access)) {
196626
196657
  set2.status = 403;
196627
- return { error: "Browser session or operator credential required" };
196658
+ return { error: "Full workspace write access required" };
196628
196659
  }
196629
196660
  const userId = ctx.userId;
196630
196661
  try {
@@ -227123,6 +227154,7 @@ var cursorListQuerySchema = exports_external2.object({
227123
227154
  sortBy: exports_external2.enum(["title", "category", "folder", "updated"]).default("updated"),
227124
227155
  sortOrder: exports_external2.enum(["asc", "desc"]).default("desc")
227125
227156
  });
227157
+ var documentIdParamsSchema = exports_external2.object({ id: exports_external2.string().uuid() });
227126
227158
  function cursorScopeHash(input) {
227127
227159
  return Buffer.from(new Bun.CryptoHasher("sha256").update(input).digest("hex"), "utf8").toString("base64url");
227128
227160
  }
@@ -227641,6 +227673,180 @@ var documentRoutes = new Elysia({ prefix: "/api" }).get("/documents", async ({ q
227641
227673
  set2.status = 500;
227642
227674
  return { error: "Failed to load pipeline progress" };
227643
227675
  }
227676
+ }).get("/documents/:id/knowledge-summary", async ({ params: params2, set: set2, request }) => {
227677
+ const parsedParams = documentIdParamsSchema.safeParse(params2);
227678
+ if (!parsedParams.success) {
227679
+ set2.status = 400;
227680
+ return { error: "Invalid document id" };
227681
+ }
227682
+ const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
227683
+ const rl = await documentRateLimiter(ip, request);
227684
+ if (!rl.allowed) {
227685
+ set2.status = 429;
227686
+ set2.headers = rateLimitHeaders(0, rl.retryAfter);
227687
+ return { error: "Too many requests" };
227688
+ }
227689
+ set2.headers = rateLimitHeaders(rl.remaining);
227690
+ const access = await resolveContentAccess(request);
227691
+ const ctx = access.ctx;
227692
+ if (ctx.role === "none") {
227693
+ set2.status = 401;
227694
+ return { error: "Unauthorized" };
227695
+ }
227696
+ if (!canAccessContent(access, "read")) {
227697
+ set2.status = 403;
227698
+ return { error: "Forbidden" };
227699
+ }
227700
+ const [summary] = await withTenant(ctx, (tx) => tx.select({
227701
+ documentId: documentKnowledgeSummaries.document_id,
227702
+ generationId: documentKnowledgeSummaries.generation_id,
227703
+ revision: documentKnowledgeSummaries.revision,
227704
+ language: documentKnowledgeSummaries.language,
227705
+ description: documentKnowledgeSummaries.description,
227706
+ keywords: documentKnowledgeSummaries.keywords,
227707
+ createdAt: documentKnowledgeSummaries.created_at,
227708
+ updatedAt: documentKnowledgeSummaries.updated_at
227709
+ }).from(documentKnowledgeSummaries).innerJoin(documents, eq(documents.id, documentKnowledgeSummaries.document_id)).where(and(eq(documentKnowledgeSummaries.document_id, parsedParams.data.id), eq(documentKnowledgeSummaries.generation_id, documents.activeEmbeddingGeneration), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1));
227710
+ if (!summary) {
227711
+ set2.status = 404;
227712
+ return { error: "Knowledge summary not found" };
227713
+ }
227714
+ return summary;
227715
+ }).get("/documents/:id/index-status", async ({ params: params2, set: set2, request }) => {
227716
+ const parsedParams = documentIdParamsSchema.safeParse(params2);
227717
+ if (!parsedParams.success) {
227718
+ set2.status = 400;
227719
+ return { error: "Invalid document id" };
227720
+ }
227721
+ const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
227722
+ const rl = await documentRateLimiter(ip, request);
227723
+ if (!rl.allowed) {
227724
+ set2.status = 429;
227725
+ set2.headers = rateLimitHeaders(0, rl.retryAfter);
227726
+ return { error: "Too many requests" };
227727
+ }
227728
+ set2.headers = rateLimitHeaders(rl.remaining);
227729
+ const access = await resolveContentAccess(request);
227730
+ const ctx = access.ctx;
227731
+ if (ctx.role === "none") {
227732
+ set2.status = 401;
227733
+ return { error: "Unauthorized" };
227734
+ }
227735
+ if (!canAccessContent(access, "read")) {
227736
+ set2.status = 403;
227737
+ return { error: "Forbidden" };
227738
+ }
227739
+ const result = await withTenant(ctx, async (tx) => {
227740
+ const [document2] = await tx.select({
227741
+ id: documents.id,
227742
+ embeddingStatus: documents.embeddingStatus,
227743
+ activeGenerationId: documents.activeEmbeddingGeneration,
227744
+ pendingGenerationId: documents.pendingEmbeddingGeneration,
227745
+ embeddingProfile: documents.embeddingProfile,
227746
+ embeddingErrorCode: documents.embeddingErrorCode,
227747
+ embeddingUpdatedAt: documents.embeddingUpdatedAt
227748
+ }).from(documents).where(and(eq(documents.id, parsedParams.data.id), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1);
227749
+ if (!document2)
227750
+ return null;
227751
+ const active = document2.activeGenerationId ? await tx.select({ id: documentEmbeddings.id }).from(documentEmbeddings).where(and(eq(documentEmbeddings.documentId, document2.id), eq(documentEmbeddings.generationId, document2.activeGenerationId), eq(documentEmbeddings.isValid, true), eq(documentEmbeddings.embeddingDimensions, 1024), eq(documentEmbeddings.embeddingProfile, document2.embeddingProfile ?? ""))).limit(1) : [];
227752
+ const [run5] = await tx.select({
227753
+ documentId: documentPipelineRuns.documentId,
227754
+ generationId: documentPipelineRuns.generationId,
227755
+ revision: documentPipelineRuns.revision,
227756
+ status: documentPipelineRuns.status,
227757
+ prepareStatus: documentPipelineRuns.prepareStatus,
227758
+ embedStatus: documentPipelineRuns.embedStatus,
227759
+ graphStatus: documentPipelineRuns.graphStatus,
227760
+ summarizeStatus: documentPipelineRuns.summarizeStatus,
227761
+ finalizeStatus: documentPipelineRuns.finalizeStatus,
227762
+ totalBatches: documentPipelineRuns.totalBatches,
227763
+ completedBatches: documentPipelineRuns.completedBatches,
227764
+ failedBatches: documentPipelineRuns.failedBatches,
227765
+ updatedAt: documentPipelineRuns.updatedAt
227766
+ }).from(documentPipelineRuns).where(eq(documentPipelineRuns.documentId, document2.id)).orderBy(desc(documentPipelineRuns.updatedAt)).limit(1);
227767
+ return {
227768
+ document: document2,
227769
+ searchable: active.length > 0,
227770
+ run: run5
227771
+ };
227772
+ });
227773
+ if (!result) {
227774
+ set2.status = 404;
227775
+ return { error: "Document not found" };
227776
+ }
227777
+ const run4 = result.run;
227778
+ return {
227779
+ documentId: result.document.id,
227780
+ embeddingStatus: result.document.embeddingStatus,
227781
+ activeGenerationId: result.document.activeGenerationId,
227782
+ pendingGenerationId: result.document.pendingGenerationId,
227783
+ embeddingProfile: result.document.embeddingProfile,
227784
+ embeddingErrorCode: result.document.embeddingErrorCode,
227785
+ embeddingUpdatedAt: result.document.embeddingUpdatedAt,
227786
+ searchable: result.searchable,
227787
+ pipeline: run4 ? {
227788
+ documentId: run4.documentId,
227789
+ generationId: run4.generationId,
227790
+ status: run4.status,
227791
+ revision: run4.revision,
227792
+ stages: {
227793
+ prepare: run4.prepareStatus,
227794
+ embed: run4.embedStatus,
227795
+ graph: run4.graphStatus,
227796
+ summarize: run4.summarizeStatus,
227797
+ finalize: run4.finalizeStatus
227798
+ },
227799
+ batches: {
227800
+ total: run4.totalBatches,
227801
+ completed: run4.completedBatches,
227802
+ failed: run4.failedBatches
227803
+ },
227804
+ updatedAt: run4.updatedAt
227805
+ } : null
227806
+ };
227807
+ }).post("/documents/:id/index/refresh", async ({ params: params2, set: set2, request }) => {
227808
+ const parsedParams = documentIdParamsSchema.safeParse(params2);
227809
+ if (!parsedParams.success) {
227810
+ set2.status = 400;
227811
+ return { error: "Invalid document id" };
227812
+ }
227813
+ const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
227814
+ const rl = await writeRateLimiter(ip, request);
227815
+ if (!rl.allowed) {
227816
+ set2.status = 429;
227817
+ set2.headers = rateLimitHeaders(0, rl.retryAfter);
227818
+ return { error: "Too many requests" };
227819
+ }
227820
+ set2.headers = rateLimitHeaders(rl.remaining);
227821
+ const access = await resolveContentAccess(request);
227822
+ const ctx = access.ctx;
227823
+ if (ctx.role === "none") {
227824
+ set2.status = 401;
227825
+ return { error: "Unauthorized" };
227826
+ }
227827
+ if (!canAccessContent(access, "edit")) {
227828
+ set2.status = 403;
227829
+ return { error: "Forbidden" };
227830
+ }
227831
+ const [document2] = await withTenant(ctx, (tx) => tx.select({
227832
+ id: documents.id,
227833
+ ownerId: documents.ownerId,
227834
+ title: documents.title,
227835
+ content: documents.content
227836
+ }).from(documents).where(and(eq(documents.id, parsedParams.data.id), tenantOwnerCondition(documents.ownerId, documents.workspaceId, ctx), isNull(documents.deletedAt), ...access.restricted && access.categoryId ? [eq(documents.categoryId, access.categoryId)] : [])).limit(1));
227837
+ if (!document2) {
227838
+ set2.status = 404;
227839
+ return { error: "Document not found" };
227840
+ }
227841
+ const queued = await enqueueDocumentPipeline({
227842
+ documentId: document2.id,
227843
+ ownerId: document2.ownerId,
227844
+ workspaceId: ctx.workspaceId,
227845
+ revision: contentHash(document2.title, document2.content ?? ""),
227846
+ source: "api"
227847
+ });
227848
+ set2.status = 202;
227849
+ return { documentId: document2.id, ...queued };
227644
227850
  }).get("/documents/:id", async ({ params: params2, set: set2, request }) => {
227645
227851
  const ip = request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? request.headers.get("x-real-ip") ?? "unknown";
227646
227852
  const rl = await documentRateLimiter(ip, request);
@@ -229073,14 +229279,15 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
229073
229279
  set2.status = 400;
229074
229280
  return { error: "Invalid query", details: parsed.error.flatten() };
229075
229281
  }
229076
- if (!config3.GRAPH_SEARCH_ENABLED) {
229077
- return { entities: [] };
229078
- }
229079
229282
  try {
229080
229283
  const allowedIds = await allowedGraphDocumentIds(access, [
229081
229284
  parsed.data.docId
229082
229285
  ]);
229083
- if (!allowedIds.has(parsed.data.docId))
229286
+ if (!allowedIds.has(parsed.data.docId)) {
229287
+ set2.status = 404;
229288
+ return { error: "Document not found" };
229289
+ }
229290
+ if (!config3.GRAPH_SEARCH_ENABLED)
229084
229291
  return { entities: [] };
229085
229292
  const entities = await fetchDocumentEntities(parsed.data.docId);
229086
229293
  return { entities };
@@ -229112,14 +229319,15 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
229112
229319
  set2.status = 400;
229113
229320
  return { error: "Invalid params", details: parsed.error.flatten() };
229114
229321
  }
229115
- if (!config3.GRAPH_SEARCH_ENABLED) {
229116
- return { related: [] };
229117
- }
229118
229322
  try {
229119
229323
  const seedIds = await allowedGraphDocumentIds(access, [
229120
229324
  parsed.data.docId
229121
229325
  ]);
229122
- if (!seedIds.has(parsed.data.docId))
229326
+ if (!seedIds.has(parsed.data.docId)) {
229327
+ set2.status = 404;
229328
+ return { error: "Document not found" };
229329
+ }
229330
+ if (!config3.GRAPH_SEARCH_ENABLED)
229123
229331
  return { related: [] };
229124
229332
  const related = await fetchRelatedDocuments(ctx, parsed.data.docId, access);
229125
229333
  return { related };
@@ -229152,12 +229360,16 @@ var graphRoutes = new Elysia({ prefix: "/api/graph" }).get("/entities", async ({
229152
229360
  return { error: "Invalid body", details: parsed.error.flatten() };
229153
229361
  }
229154
229362
  const { query, docIds, maxResults } = parsed.data;
229155
- if (!config3.GRAPH_SEARCH_ENABLED) {
229156
- return { query, entities: [], relatedDocs: [] };
229157
- }
229158
229363
  try {
229159
229364
  const authorizedSeeds = await allowedGraphDocumentIds(access, docIds);
229160
- const result = await graphRagLookup(ctx, docIds.filter((id3) => authorizedSeeds.has(id3)), maxResults, access);
229365
+ if (docIds.some((id3) => !authorizedSeeds.has(id3))) {
229366
+ set2.status = 404;
229367
+ return { error: "One or more documents were not found" };
229368
+ }
229369
+ if (!config3.GRAPH_SEARCH_ENABLED) {
229370
+ return { query, entities: [], relatedDocs: [] };
229371
+ }
229372
+ const result = await graphRagLookup(ctx, docIds, maxResults, access);
229161
229373
  return { query, ...result };
229162
229374
  } catch (err) {
229163
229375
  logger3.warn({ err, docIds: docIds.length }, "Graph RAG search failed \u2014 returning empty");
@@ -230229,7 +230441,6 @@ async function retrieveVector(ctx, plan, limit, minimum, chunkLimit, execute, pr
230229
230441
  JOIN documents d ON d.id = de.document_id
230230
230442
  WHERE d.owner_id = ${ctx.userId}
230231
230443
  ${scope}
230232
- AND d.embedding_status = 'ready'
230233
230444
  AND d.active_embedding_generation IS NOT NULL
230234
230445
  AND de.generation_id = d.active_embedding_generation
230235
230446
  AND de.is_valid = true
@@ -230961,7 +231172,6 @@ async function hydrateResults(ctx, items, includeChunks, query, allowedDocumentI
230961
231172
  JOIN documents d ON d.id = de.document_id
230962
231173
  WHERE de.document_id IN (${sql.join(ids.map((id3) => sql`${id3}`), sql`, `)})
230963
231174
  AND ${tenantOwnerSql("d", ctx)}
230964
- AND d.embedding_status = 'ready'
230965
231175
  AND d.active_embedding_generation IS NOT NULL
230966
231176
  AND de.generation_id = d.active_embedding_generation
230967
231177
  AND de.is_valid = true
@@ -232729,6 +232939,116 @@ async function ensureApiKeyOwner() {
232729
232939
  init_config();
232730
232940
  init_logger3();
232731
232941
 
232942
+ // ../../backend/src/lib/reembed-cron.ts
232943
+ init_schema2();
232944
+ init_drizzle_orm();
232945
+ init_config();
232946
+
232947
+ // ../../backend/src/lib/cron-timer-registry.ts
232948
+ function createCronTimerRegistry(clear = clearInterval) {
232949
+ const handles = [];
232950
+ let closed = false;
232951
+ return {
232952
+ register(handle) {
232953
+ if (closed) {
232954
+ clear(handle);
232955
+ return;
232956
+ }
232957
+ handles.push(handle);
232958
+ },
232959
+ close() {
232960
+ if (closed)
232961
+ return;
232962
+ closed = true;
232963
+ for (const handle of handles)
232964
+ clear(handle);
232965
+ handles.length = 0;
232966
+ }
232967
+ };
232968
+ }
232969
+
232970
+ // ../../backend/src/lib/reembed-cron.ts
232971
+ init_logger3();
232972
+ var METADATA_DEBOUNCE_MINUTES = 3;
232973
+ var CRON_BATCH_SIZE = 100;
232974
+ var CRON_TENANT = adminTenantContext(ZERO_UUID);
232975
+ function startReembedCron() {
232976
+ const timers = createCronTimerRegistry();
232977
+ const metadataIntervalMs = config3.METADATA_REEMBED_CRON_INTERVAL_MINUTES * 60 * 1000;
232978
+ if (metadataIntervalMs > 0) {
232979
+ logger3.info({ intervalMinutes: config3.METADATA_REEMBED_CRON_INTERVAL_MINUTES }, "Reembed metadata-stale cron started");
232980
+ processStaleMetadataChanges().catch((err) => {
232981
+ logger3.error({ err }, "Reembed metadata-stale cron initial run failed");
232982
+ });
232983
+ timers.register(setInterval(() => {
232984
+ processStaleMetadataChanges().catch((err) => {
232985
+ logger3.error({ err }, "Reembed metadata-stale cron tick failed");
232986
+ });
232987
+ }, metadataIntervalMs));
232988
+ } else {
232989
+ logger3.warn("METADATA_REEMBED_CRON_INTERVAL_MINUTES=0 \u2014 metadata-stale cron is DISABLED. " + "Metadata-only changes will NOT trigger re-embed until the cron is re-enabled.");
232990
+ }
232991
+ const idleIntervalMs = config3.REEMBED_CRON_INTERVAL_MINUTES * 60 * 1000;
232992
+ if (idleIntervalMs > 0) {
232993
+ logger3.info({ intervalMinutes: config3.REEMBED_CRON_INTERVAL_MINUTES }, "Reembed idle-pending cron started");
232994
+ processIdlePendingChanges().catch((err) => {
232995
+ logger3.error({ err }, "Reembed idle-pending cron initial run failed");
232996
+ });
232997
+ timers.register(setInterval(() => {
232998
+ processIdlePendingChanges().catch((err) => {
232999
+ logger3.error({ err }, "Reembed idle-pending cron tick failed");
233000
+ });
233001
+ }, idleIntervalMs));
233002
+ } else {
233003
+ logger3.warn("REEMBED_CRON_INTERVAL_MINUTES=0 \u2014 idle-pending cron is DISABLED. " + "Sub-threshold edits will NOT trigger catch-up embeds until the cron is re-enabled.");
233004
+ }
233005
+ return { close: () => timers.close() };
233006
+ }
233007
+ async function processStaleMetadataChanges() {
233008
+ const cutoff = new Date(Date.now() - METADATA_DEBOUNCE_MINUTES * 60 * 1000);
233009
+ const stale = await withTenant(CRON_TENANT, (tx) => tx.select({
233010
+ id: documents.id,
233011
+ workspaceId: documents.workspaceId,
233012
+ metadataChangedAt: documents.metadataChangedAt
233013
+ }).from(documents).where(and(isNotNull(documents.metadataChangedAt), lt(documents.metadataChangedAt, cutoff))).orderBy(documents.metadataChangedAt).limit(CRON_BATCH_SIZE));
233014
+ if (stale.length === 0)
233015
+ return;
233016
+ logger3.debug({ count: stale.length }, "Reembed cron: processing stale metadata changes");
233017
+ let enqueued = 0;
233018
+ for (const row of stale) {
233019
+ const originalTimestamp = row.metadataChangedAt instanceof Date ? row.metadataChangedAt.toISOString() : String(row.metadataChangedAt);
233020
+ const cleared = await withTenant(CRON_TENANT, (tx) => tx.execute(sql`
233021
+ UPDATE documents
233022
+ SET metadata_changed_at = NULL
233023
+ WHERE id = ${row.id}
233024
+ AND metadata_changed_at = ${originalTimestamp}
233025
+ RETURNING id
233026
+ `));
233027
+ if (Array.isArray(cleared) && cleared.length > 0) {
233028
+ await enqueueReembed([row.id], row.workspaceId ?? undefined);
233029
+ enqueued += 1;
233030
+ }
233031
+ }
233032
+ if (enqueued > 0) {
233033
+ logger3.info({ scanned: stale.length, enqueued }, "Reembed cron: enqueued stale metadata changes");
233034
+ }
233035
+ }
233036
+ async function processIdlePendingChanges() {
233037
+ const cutoff = new Date(Date.now() - config3.REEMBED_MAX_IDLE_HOURS * 3600000);
233038
+ const idle = await withTenant(CRON_TENANT, (tx) => tx.select({ id: documents.id, workspaceId: documents.workspaceId }).from(documents).where(and(eq(documents.pendingMinorChanges, true), lt(documents.lastSignificantUpdateAt, cutoff))).orderBy(documents.lastSignificantUpdateAt).limit(CRON_BATCH_SIZE));
233039
+ if (idle.length === 0)
233040
+ return;
233041
+ logger3.debug({ count: idle.length }, "Reembed cron: processing idle pending-minor changes");
233042
+ let enqueued = 0;
233043
+ for (const row of idle) {
233044
+ await enqueueReembed([row.id], row.workspaceId ?? undefined);
233045
+ enqueued += 1;
233046
+ }
233047
+ if (enqueued > 0) {
233048
+ logger3.info({ scanned: idle.length, enqueued }, "Reembed cron: enqueued idle pending-minor changes");
233049
+ }
233050
+ }
233051
+
232732
233052
  // ../../backend/src/queue/adapters.ts
232733
233053
  init_schema2();
232734
233054
  init_drizzle_orm();
@@ -232782,15 +233102,62 @@ async function activateEmbeddingGeneration(documentId, generationId, expectedChu
232782
233102
  // ../../backend/src/queue/adapters.ts
232783
233103
  init_config();
232784
233104
 
233105
+ // ../../backend/src/lib/graph/generation-state.ts
233106
+ function literal2(value) {
233107
+ return JSON.stringify(value);
233108
+ }
233109
+ function isStaleRevisionError(error53) {
233110
+ return error53 instanceof Error && error53.name === "stale_revision";
233111
+ }
233112
+ async function runGenerationFencedGraphWrite(dependencies) {
233113
+ if (!await dependencies.lockCurrentGeneration()) {
233114
+ const error53 = new Error("graph generation is stale");
233115
+ error53.name = "stale_revision";
233116
+ throw error53;
233117
+ }
233118
+ await dependencies.persist();
233119
+ }
233120
+ function graphReplacementCyphers(identity) {
233121
+ const documentId = literal2(identity.documentId);
233122
+ const generationId = literal2(identity.generationId);
233123
+ const revision = literal2(identity.revision);
233124
+ const timestamp3 = literal2(identity.timestamp);
233125
+ return [
233126
+ `MATCH (d:Document {id: ${documentId}})-[legacy:MENTIONS]->()
233127
+ DELETE legacy RETURN count(legacy)`,
233128
+ `MATCH ()-[r]->() WHERE r.document_id = ${documentId}
233129
+ DELETE r RETURN count(r)`,
233130
+ `MERGE (d:Document {id: ${documentId}})
233131
+ SET d.generation_id = ${generationId}, d.revision = ${revision},
233132
+ d.created_at = coalesce(d.created_at, ${timestamp3}),
233133
+ d.entity_extracted_at = ${timestamp3}
233134
+ RETURN d.id`
233135
+ ];
233136
+ }
233137
+ function graphCompensationCyphers(documentIdValue, generationIdValue) {
233138
+ const documentId = literal2(documentIdValue);
233139
+ const generationId = literal2(generationIdValue);
233140
+ return [
233141
+ `MATCH ()-[r]->()
233142
+ WHERE r.document_id = ${documentId}
233143
+ AND r.generation_id = ${generationId}
233144
+ DELETE r RETURN count(r)`,
233145
+ `MATCH (d:Document {id: ${documentId}})
233146
+ WHERE d.generation_id = ${generationId}
233147
+ DETACH DELETE d RETURN count(d)`
233148
+ ];
233149
+ }
233150
+
232785
233151
  // ../../backend/src/lib/graph/delete-document-state.ts
232786
- async function deleteDocumentGraphState(documentId) {
233152
+ async function deleteDocumentGraphGeneration(documentId, generationId) {
232787
233153
  const sql6 = await getGraphDb();
232788
233154
  if (!sql6)
232789
233155
  return;
232790
- const literal2 = JSON.stringify(documentId);
232791
233156
  await sql6.begin(async (tx) => {
232792
233157
  await tx.unsafe("SET LOCAL search_path = ag_catalog, public");
232793
- await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ MATCH (d:Document {id: ${literal2}}) DETACH DELETE d RETURN 1 $$) AS (deleted agtype)`);
233158
+ for (const cypher of graphCompensationCyphers(documentId, generationId)) {
233159
+ await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (deleted agtype)`);
233160
+ }
232794
233161
  });
232795
233162
  }
232796
233163
 
@@ -232813,8 +233180,9 @@ var RELATION_TYPES = [
232813
233180
  "AUTHORED_BY"
232814
233181
  ];
232815
233182
  var EXTRACT_DEDUP_TTL_SECONDS = 24 * 60 * 60;
232816
- function extractDedupKey(documentId, chunkIndex, chunkHash2) {
232817
- return `hiai-docs:extract:done:${documentId}:${chunkIndex}:${chunkHash2}`;
233183
+ function extractDedupKey(documentId, chunkIndex, chunkHash2, generationId) {
233184
+ const generation = generationId ? `${generationId}:` : "";
233185
+ return `hiai-docs:extract:done:${documentId}:${generation}${chunkIndex}:${chunkHash2}`;
232818
233186
  }
232819
233187
  var DEFAULT_MAX_TOKENS = 1024;
232820
233188
  var DEFAULT_TEMPERATURE = 0;
@@ -232835,12 +233203,24 @@ function setCachedEntity(name, type, confidence) {
232835
233203
  async function extractEntities(chunkText2, documentId, options = {}) {
232836
233204
  if (!config3.GRAPH_EXTRACT_ENABLED)
232837
233205
  return [];
233206
+ let claimedDedupKey;
233207
+ const releaseDedupClaim = async () => {
233208
+ if (!claimedDedupKey)
233209
+ return;
233210
+ try {
233211
+ await redis.del(claimedDedupKey);
233212
+ } catch (err) {
233213
+ logger3.warn({ err, documentId, claimedDedupKey }, "Failed to release extract-dedup slot");
233214
+ }
233215
+ };
232838
233216
  if (options.chunkHash && options.chunkIndex !== undefined) {
232839
- const key = extractDedupKey(documentId, options.chunkIndex, options.chunkHash);
233217
+ const key = extractDedupKey(documentId, options.chunkIndex, options.chunkHash, options.generationId);
232840
233218
  let acquired = false;
232841
233219
  try {
232842
233220
  const result = await redis.set(key, "1", "EX", EXTRACT_DEDUP_TTL_SECONDS, "NX");
232843
233221
  acquired = result === "OK";
233222
+ if (acquired)
233223
+ claimedDedupKey = key;
232844
233224
  } catch (err) {
232845
233225
  logger3.warn({ err, documentId, chunkIndex: options.chunkIndex }, "Extract-dedup Redis SET failed \u2014 falling through to extraction");
232846
233226
  acquired = true;
@@ -232849,22 +233229,31 @@ async function extractEntities(chunkText2, documentId, options = {}) {
232849
233229
  return [];
232850
233230
  }
232851
233231
  const sql6 = await getGraphDb();
232852
- if (!sql6)
233232
+ if (!sql6) {
233233
+ await releaseDedupClaim();
232853
233234
  return [];
232854
- if (!chunkText2)
233235
+ }
233236
+ if (!chunkText2) {
233237
+ await releaseDedupClaim();
232855
233238
  return [];
233239
+ }
232856
233240
  let entities;
232857
233241
  try {
232858
233242
  entities = await callEntityExtractionLLM(chunkText2, options);
232859
233243
  } catch (err) {
233244
+ await releaseDedupClaim();
232860
233245
  logger3.warn({ err, documentId }, "Entity extraction LLM call failed \u2014 skipping");
232861
233246
  return [];
232862
233247
  }
232863
- if (entities.length === 0)
232864
- return [];
232865
233248
  try {
232866
- await persistEntities(sql6, documentId, entities);
233249
+ await persistEntities(sql6, documentId, entities, {
233250
+ generationId: options.generationId ?? "legacy",
233251
+ revision: options.revision ?? "legacy"
233252
+ });
232867
233253
  } catch (err) {
233254
+ await releaseDedupClaim();
233255
+ if (isStaleRevisionError(err))
233256
+ throw err;
232868
233257
  logger3.warn({ err, documentId, count: entities.length }, "Failed to persist extracted entities to AGE \u2014 discarding");
232869
233258
  return [];
232870
233259
  }
@@ -233033,30 +233422,55 @@ function stripMarkdownFences(text4) {
233033
233422
  const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(text4.trim());
233034
233423
  return fenced?.[1] ?? text4;
233035
233424
  }
233036
- async function persistEntities(sql6, documentId, entities) {
233425
+ async function persistEntities(sql6, documentId, entities, identity) {
233037
233426
  await sql6.begin(async (tx) => {
233038
233427
  await tx`SELECT pg_catalog.set_config('search_path', 'ag_catalog, "$user", public', false)`;
233428
+ await tx`SELECT pg_catalog.set_config('app.current_user_id', '00000000-0000-0000-0000-000000000000', true)`;
233429
+ await tx`SELECT pg_catalog.set_config('app.current_user_role', 'admin', true)`;
233430
+ await tx`SELECT pg_catalog.set_config('app.current_workspace_id', '', true)`;
233039
233431
  const nowIso = new Date().toISOString();
233040
- const docIdLiteral = JSON.stringify(documentId);
233041
- const nowLiteral = JSON.stringify(nowIso);
233042
- await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$
233043
- MERGE (d:Document {id: ${docIdLiteral}})
233044
- SET d.created_at = ${nowLiteral}, d.entity_extracted_at = ${nowLiteral}
233045
- RETURN d.id
233046
- $$) AS (result agtype)`);
233047
- for (const ent of entities) {
233048
- const label = ent.type;
233049
- const name = ent.name;
233050
- const conf = ent.confidence;
233051
- await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${entityUpsertCypher(documentId, label, name, nowIso, conf)} $$) AS (result agtype)`);
233052
- await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${documentEntityEdgeCypher(documentId, label, name, conf)} $$) AS (result agtype)`);
233053
- }
233054
- for (const ent of entities) {
233055
- for (const rel of ent.relationships) {
233056
- const cypher = entityRelationCypher(ent.type, ent.name, rel.targetName, rel.relationType, rel.confidence);
233057
- await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
233432
+ await runGenerationFencedGraphWrite({
233433
+ async lockCurrentGeneration() {
233434
+ const rows = await tx`
233435
+ SELECT d.id
233436
+ FROM public.documents d
233437
+ JOIN public.document_pipeline_runs r
233438
+ ON r.document_id = d.id
233439
+ AND r.generation_id = ${identity.generationId}::uuid
233440
+ WHERE d.id = ${documentId}::uuid
233441
+ AND d.active_embedding_generation = ${identity.generationId}::uuid
233442
+ AND d.content_hash = ${identity.revision}
233443
+ AND r.revision = ${identity.revision}
233444
+ AND r.status NOT IN ('cancelled', 'failed')
233445
+ AND r.graph_status = 'processing'
233446
+ FOR UPDATE OF d, r
233447
+ `;
233448
+ return rows.length === 1;
233449
+ },
233450
+ async persist() {
233451
+ for (const cypher of graphReplacementCyphers({
233452
+ documentId,
233453
+ generationId: identity.generationId,
233454
+ revision: identity.revision,
233455
+ timestamp: nowIso
233456
+ })) {
233457
+ await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
233458
+ }
233459
+ for (const ent of entities) {
233460
+ const label = ent.type;
233461
+ const name = ent.name;
233462
+ const conf = ent.confidence;
233463
+ await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${entityUpsertCypher(documentId, label, name, nowIso, conf)} $$) AS (result agtype)`);
233464
+ await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${documentEntityEdgeCypher(documentId, identity.generationId, label, name, conf)} $$) AS (result agtype)`);
233465
+ }
233466
+ for (const ent of entities) {
233467
+ for (const rel of ent.relationships) {
233468
+ const cypher = entityRelationCypher(documentId, identity.generationId, ent.type, ent.name, rel.targetName, rel.relationType, rel.confidence);
233469
+ await tx.unsafe(`SELECT * FROM cypher('docs_graph', $$ ${cypher} $$) AS (result agtype)`);
233470
+ }
233471
+ }
233058
233472
  }
233059
- }
233473
+ });
233060
233474
  });
233061
233475
  }
233062
233476
  function entityUpsertCypher(docId, label, name, nowIso, confidence) {
@@ -233065,27 +233479,118 @@ function entityUpsertCypher(docId, label, name, nowIso, confidence) {
233065
233479
  MERGE (e:\`${label}\` {name: $name})
233066
233480
  SET e.created_at = $now, e.last_seen_doc = $docId, e.confidence = $conf
233067
233481
  RETURN e.name
233068
- `.replace("$name", JSON.stringify(name)).replace("$now", JSON.stringify(nowIso)).replace("$docId", JSON.stringify(docId)).replace("$conf", confLiteral);
233482
+ `.replaceAll("$name", JSON.stringify(name)).replaceAll("$now", JSON.stringify(nowIso)).replaceAll("$docId", JSON.stringify(docId)).replaceAll("$conf", confLiteral);
233069
233483
  }
233070
- function documentEntityEdgeCypher(docId, label, name, confidence) {
233484
+ function documentEntityEdgeCypher(docId, generationId, label, name, confidence) {
233071
233485
  const confLiteral = typeof confidence === "number" ? JSON.stringify(confidence) : "null";
233072
233486
  return `
233073
233487
  MATCH (d:Document {id: $docId})
233074
233488
  MATCH (e:\`${label}\` {name: $name})
233075
- MERGE (d)-[r:MENTIONS]->(e)
233076
- SET r.created_at = $now, r.confidence = $conf
233489
+ MERGE (d)-[r:MENTIONS {document_id: $docId}]->(e)
233490
+ SET r.created_at = $now, r.confidence = $conf, r.generation_id = $generationId
233077
233491
  RETURN r
233078
- `.replace("$docId", JSON.stringify(docId)).replace("$name", JSON.stringify(name)).replace("$now", JSON.stringify(new Date().toISOString())).replace("$conf", confLiteral);
233492
+ `.replaceAll("$docId", JSON.stringify(docId)).replaceAll("$name", JSON.stringify(name)).replaceAll("$now", JSON.stringify(new Date().toISOString())).replaceAll("$generationId", JSON.stringify(generationId)).replaceAll("$conf", confLiteral);
233079
233493
  }
233080
- function entityRelationCypher(sourceLabel, sourceName, targetName, relationType, confidence) {
233494
+ function entityRelationCypher(documentId, generationId, sourceLabel, sourceName, targetName, relationType, confidence) {
233081
233495
  const confLiteral = typeof confidence === "number" ? JSON.stringify(confidence) : "null";
233082
233496
  return `
233083
233497
  MATCH (a:\`${sourceLabel}\` {name: $source})
233084
233498
  MATCH (b {name: $target})
233085
- MERGE (a)-[r:\`${relationType}\`]->(b)
233086
- SET r.created_at = $now, r.confidence = $conf
233499
+ MERGE (a)-[r:\`${relationType}\` {document_id: $documentId}]->(b)
233500
+ SET r.created_at = $now, r.confidence = $conf, r.generation_id = $generationId
233087
233501
  RETURN r
233088
- `.replace("$source", JSON.stringify(sourceName)).replace("$target", JSON.stringify(targetName)).replace("$now", JSON.stringify(new Date().toISOString())).replace("$conf", confLiteral);
233502
+ `.replaceAll("$source", JSON.stringify(sourceName)).replaceAll("$target", JSON.stringify(targetName)).replaceAll("$documentId", JSON.stringify(documentId)).replaceAll("$generationId", JSON.stringify(generationId)).replaceAll("$now", JSON.stringify(new Date().toISOString())).replaceAll("$conf", confLiteral);
233503
+ }
233504
+
233505
+ // ../../backend/src/lib/knowledge-summary.ts
233506
+ async function runKnowledgeSummaryStage(dependencies) {
233507
+ const document2 = await dependencies.readCurrent();
233508
+ if (!document2)
233509
+ return "cancelled";
233510
+ const summary = await dependencies.generate(document2);
233511
+ if (summary.status === "skipped")
233512
+ return "skipped";
233513
+ return await dependencies.persistIfCurrent(summary) ? "ready" : "cancelled";
233514
+ }
233515
+ async function buildKnowledgeSummary(document2, provider) {
233516
+ if (!document2.content.trim()) {
233517
+ return { status: "skipped", reason: "empty_document" };
233518
+ }
233519
+ const output = await provider(document2);
233520
+ if (!output)
233521
+ throw new Error("summary_provider_failed");
233522
+ const language = output.language.trim();
233523
+ const description = output.description.trim();
233524
+ if (!language || !description)
233525
+ throw new Error("summary_provider_invalid");
233526
+ const seen = new Set;
233527
+ const keywords = [];
233528
+ for (const raw of output.keywords) {
233529
+ const keyword = raw.trim();
233530
+ const key = keyword.toLocaleLowerCase("en");
233531
+ if (!keyword || seen.has(key))
233532
+ continue;
233533
+ seen.add(key);
233534
+ keywords.push(keyword);
233535
+ if (keywords.length === 20)
233536
+ break;
233537
+ }
233538
+ return {
233539
+ status: "ready",
233540
+ language,
233541
+ description,
233542
+ keywords,
233543
+ model: output.model
233544
+ };
233545
+ }
233546
+
233547
+ // ../../backend/src/lib/knowledge-summary-provider.ts
233548
+ init_zod();
233549
+ init_config();
233550
+ var summarySchema = exports_external2.object({
233551
+ language: exports_external2.string().min(1).max(32),
233552
+ description: exports_external2.string().min(1).max(2000),
233553
+ keywords: exports_external2.array(exports_external2.string().min(1).max(100)).max(20)
233554
+ });
233555
+ var systemPrompt = [
233556
+ "Summarize a knowledge-base document for retrieval metadata.",
233557
+ "Return only JSON with language, description, and keywords.",
233558
+ "Use a concise factual description and at most 20 distinct keywords."
233559
+ ].join(`
233560
+ `);
233561
+ function provider(baseUrl, apiKey, model) {
233562
+ if (!baseUrl || !model)
233563
+ return;
233564
+ return {
233565
+ baseUrl,
233566
+ model,
233567
+ apiKey: resolveChatProviderKey(baseUrl, apiKey, config3.OPENROUTER_API_KEY),
233568
+ timeoutMs: config3.GRAPH_EXTRACT_TIMEOUT_MS,
233569
+ reasoningEffort: config3.GRAPH_EXTRACT_REASONING_EFFORT
233570
+ };
233571
+ }
233572
+ async function requestKnowledgeSummary(document2) {
233573
+ const primary = provider(config3.GRAPH_EXTRACT_BASE_URL, config3.GRAPH_EXTRACT_API_KEY, config3.GRAPH_EXTRACT_MODEL);
233574
+ if (!primary)
233575
+ return null;
233576
+ const fallback = provider(config3.GRAPH_EXTRACT_FALLBACK_BASE_URL, config3.GRAPH_EXTRACT_FALLBACK_API_KEY, config3.GRAPH_EXTRACT_FALLBACK_MODEL);
233577
+ const result = await requestStructuredChat({
233578
+ primary,
233579
+ fallback,
233580
+ messages: [
233581
+ { role: "system", content: systemPrompt },
233582
+ {
233583
+ role: "user",
233584
+ content: `Title: ${document2.title}
233585
+
233586
+ ${document2.content.slice(0, 50000)}`
233587
+ }
233588
+ ],
233589
+ outputSchema: summarySchema,
233590
+ maxTokens: 768,
233591
+ temperature: 0
233592
+ });
233593
+ return result ? { ...result.data, model: result.model } : null;
233089
233594
  }
233090
233595
 
233091
233596
  // ../../backend/src/queue/document-revision.ts
@@ -233263,6 +233768,23 @@ async function getRun(generationId) {
233263
233768
  return run4 ?? null;
233264
233769
  });
233265
233770
  }
233771
+ async function isCurrentPipelineGeneration(job2) {
233772
+ return withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
233773
+ const [current] = await tx.select({ id: documents.id }).from(documents).innerJoin(documentPipelineRuns, and(eq(documentPipelineRuns.documentId, documents.id), eq(documentPipelineRuns.generationId, job2.generationId))).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documentPipelineRuns.revision, job2.revision), ne(documentPipelineRuns.status, "cancelled"), ne(documentPipelineRuns.status, "failed"), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
233774
+ return Boolean(current);
233775
+ });
233776
+ }
233777
+ async function cancelStalePipelineRun(job2) {
233778
+ await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
233779
+ status: "cancelled",
233780
+ graphStatus: "cancelled",
233781
+ summarizeStatus: "cancelled",
233782
+ finalizeStatus: "cancelled",
233783
+ errorCode: "stale_revision",
233784
+ completedAt: new Date,
233785
+ updatedAt: new Date
233786
+ }).where(and(eq(documentPipelineRuns.generationId, job2.generationId), ne(documentPipelineRuns.status, "cancelled"))));
233787
+ }
233266
233788
  async function setStageStatus(generationId, stage, status2, errorCode) {
233267
233789
  await withTenant(admin, (tx) => tx.update(documentPipelineRuns).set({
233268
233790
  ...stagePatch(stage, status2),
@@ -233518,18 +234040,22 @@ function createPipelineStageDependencies(redisUrl) {
233518
234040
  },
233519
234041
  async extract(job2) {
233520
234042
  const doc5 = await withTenant(admin, async (tx) => {
233521
- const [row] = await tx.select({ content: documents.content }).from(documents).where(and(eq(documents.id, job2.documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
234043
+ const [row] = await tx.select({ content: documents.content }).from(documents).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documents.contentHash, job2.revision), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
233522
234044
  return row;
233523
234045
  });
233524
234046
  if (!doc5)
233525
234047
  throw new Error("Pipeline document not found");
233526
- await withProviderPermit(providerProfile(`graph:${config3.GRAPH_EXTRACT_MODEL ?? "default"}`), "graph", () => extractEntities(doc5.content ?? "", job2.documentId));
234048
+ await withProviderPermit(providerProfile(`graph:${config3.GRAPH_EXTRACT_MODEL ?? "default"}`), "graph", () => extractEntities(doc5.content ?? "", job2.documentId, {
234049
+ generationId: job2.generationId,
234050
+ revision: job2.revision
234051
+ }));
233527
234052
  },
233528
234053
  async compensateExtract(job2) {
233529
234054
  const owned = await withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => tx.select({ id: documents.id }).from(documents).where(and(eq(documents.id, job2.documentId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1));
233530
234055
  if (owned.length === 1)
233531
- await deleteDocumentGraphState(job2.documentId);
234056
+ await deleteDocumentGraphGeneration(job2.documentId, job2.generationId);
233532
234057
  },
234058
+ cancelStaleRun: cancelStalePipelineRun,
233533
234059
  setGraphStatus: (generationId, status2, errorCode) => setStageStatus(generationId, "graph", status2, errorCode),
233534
234060
  async enqueueSummarize(job2) {
233535
234061
  const data = { ...job2, stage: "summarize" };
@@ -233542,6 +234068,8 @@ function createPipelineStageDependencies(redisUrl) {
233542
234068
  },
233543
234069
  summarize: {
233544
234070
  isCancelled: async (job2) => (await getRun(job2.generationId))?.status === "cancelled",
234071
+ isCurrent: isCurrentPipelineGeneration,
234072
+ cancelStaleRun: cancelStalePipelineRun,
233545
234073
  async getRun(job2) {
233546
234074
  const run4 = await getRun(job2.generationId);
233547
234075
  return run4 ? {
@@ -233552,8 +234080,59 @@ function createPipelineStageDependencies(redisUrl) {
233552
234080
  embedStatus: pipelineStageStatus(run4.embedStatus)
233553
234081
  } : null;
233554
234082
  },
233555
- enabled: () => false,
233556
- async summarize() {},
234083
+ enabled: () => config3.GRAPH_EXTRACT_ENABLED && Boolean(config3.GRAPH_EXTRACT_BASE_URL && config3.GRAPH_EXTRACT_MODEL),
234084
+ async summarize(job2) {
234085
+ return runKnowledgeSummaryStage({
234086
+ readCurrent: () => withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
234087
+ const [document2] = await tx.select({
234088
+ title: documents.title,
234089
+ content: documents.content,
234090
+ contentHash: documents.contentHash,
234091
+ activeGeneration: documents.activeEmbeddingGeneration
234092
+ }).from(documents).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1);
234093
+ if (!document2)
234094
+ return null;
234095
+ const revision = resolveDocumentRevision(document2.contentHash, document2.title, document2.content ?? "");
234096
+ if (revision !== job2.revision)
234097
+ return null;
234098
+ return {
234099
+ title: document2.title,
234100
+ content: document2.content ?? "",
234101
+ revision
234102
+ };
234103
+ }),
234104
+ generate: (document2) => buildKnowledgeSummary(document2, requestKnowledgeSummary),
234105
+ persistIfCurrent: (summary) => withTenant({ ...admin, workspaceId: job2.workspaceId }, async (tx) => {
234106
+ const [current] = await tx.select({ id: documents.id }).from(documents).innerJoin(documentPipelineRuns, and(eq(documentPipelineRuns.documentId, documents.id), eq(documentPipelineRuns.generationId, job2.generationId))).where(and(eq(documents.id, job2.documentId), eq(documents.activeEmbeddingGeneration, job2.generationId), eq(documents.contentHash, job2.revision), eq(documentPipelineRuns.revision, job2.revision), eq(documentPipelineRuns.summarizeStatus, "processing"), ne(documentPipelineRuns.status, "cancelled"), ne(documentPipelineRuns.status, "failed"), tenantOwnerCondition(documents.ownerId, documents.workspaceId, jobTenant(job2)))).limit(1).for("update");
234107
+ if (!current)
234108
+ return false;
234109
+ await tx.insert(documentKnowledgeSummaries).values({
234110
+ document_id: job2.documentId,
234111
+ owner_id: job2.ownerId,
234112
+ workspace_id: job2.workspaceId,
234113
+ generation_id: job2.generationId,
234114
+ revision: job2.revision,
234115
+ language: summary.language,
234116
+ description: summary.description,
234117
+ keywords: summary.keywords,
234118
+ updated_at: new Date
234119
+ }).onConflictDoUpdate({
234120
+ target: [
234121
+ documentKnowledgeSummaries.document_id,
234122
+ documentKnowledgeSummaries.generation_id
234123
+ ],
234124
+ set: {
234125
+ revision: job2.revision,
234126
+ language: summary.language,
234127
+ description: summary.description,
234128
+ keywords: summary.keywords,
234129
+ updated_at: new Date
234130
+ }
234131
+ });
234132
+ return true;
234133
+ })
234134
+ });
234135
+ },
233557
234136
  setSummaryStatus: (generationId, status2, errorCode) => setStageStatus(generationId, "summarize", status2, errorCode),
233558
234137
  async enqueueFinalize(job2) {
233559
234138
  const data = { ...job2, stage: "finalize" };
@@ -233938,10 +234517,8 @@ function createEmbedWorker(redisUrl, deps, options = {}) {
233938
234517
  });
233939
234518
  }
233940
234519
 
233941
- // ../../backend/src/queue/workers/finalize.worker.ts
234520
+ // ../../backend/src/queue/stage-policies.ts
233942
234521
  function deriveFinalStatus(run4) {
233943
- if (!run4)
233944
- throw new Error("Pipeline run not found");
233945
234522
  if (run4.status === "cancelled")
233946
234523
  return "cancelled";
233947
234524
  if (run4.embedStatus === "failed" || run4.embedStatus === "cancelled")
@@ -233956,6 +234533,96 @@ function deriveFinalStatus(run4) {
233956
234533
  }
233957
234534
  return "failed";
233958
234535
  }
234536
+ async function processSummaryStage(job2, dependencies) {
234537
+ const isExplicitlyCancelled = async () => await dependencies.isCancelled?.(job2) === true;
234538
+ const hasCurrentGeneration = async () => await dependencies.isCurrent?.(job2) !== false;
234539
+ const continueIfCurrent = async () => {
234540
+ if (await isExplicitlyCancelled())
234541
+ return false;
234542
+ if (await hasCurrentGeneration())
234543
+ return true;
234544
+ await dependencies.cancelStaleRun?.(job2);
234545
+ return false;
234546
+ };
234547
+ if (!await continueIfCurrent())
234548
+ return;
234549
+ const run4 = await dependencies.getRun(job2);
234550
+ if (!run4)
234551
+ throw new Error("Pipeline run not found");
234552
+ if (run4.ownerId !== job2.ownerId || run4.documentId !== job2.documentId) {
234553
+ throw new Error("Pipeline owner mismatch");
234554
+ }
234555
+ if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
234556
+ await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
234557
+ await dependencies.cancelStaleRun?.(job2);
234558
+ return;
234559
+ }
234560
+ if (!dependencies.enabled()) {
234561
+ if (!await continueIfCurrent())
234562
+ return;
234563
+ await dependencies.setSummaryStatus(job2.generationId, "skipped");
234564
+ if (!await continueIfCurrent())
234565
+ return;
234566
+ await dependencies.enqueueFinalize(job2);
234567
+ return;
234568
+ }
234569
+ if (!await continueIfCurrent())
234570
+ return;
234571
+ await dependencies.setSummaryStatus(job2.generationId, "processing");
234572
+ try {
234573
+ if (!await continueIfCurrent())
234574
+ return;
234575
+ const status2 = await dependencies.summarize(job2);
234576
+ if (status2 === "cancelled") {
234577
+ if (await isExplicitlyCancelled()) {
234578
+ await dependencies.setSummaryStatus(job2.generationId, "cancelled");
234579
+ return;
234580
+ }
234581
+ await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
234582
+ await dependencies.cancelStaleRun?.(job2);
234583
+ return;
234584
+ }
234585
+ if (await isExplicitlyCancelled()) {
234586
+ await dependencies.setSummaryStatus(job2.generationId, "cancelled");
234587
+ return;
234588
+ }
234589
+ if (!await hasCurrentGeneration()) {
234590
+ if (await isExplicitlyCancelled())
234591
+ return;
234592
+ await dependencies.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
234593
+ await dependencies.cancelStaleRun?.(job2);
234594
+ return;
234595
+ }
234596
+ await dependencies.setSummaryStatus(job2.generationId, status2);
234597
+ } catch (error53) {
234598
+ if (!await continueIfCurrent())
234599
+ return;
234600
+ await dependencies.setSummaryStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "summary_failed");
234601
+ if (!await continueIfCurrent())
234602
+ return;
234603
+ await dependencies.enqueueFinalize(job2);
234604
+ return;
234605
+ }
234606
+ if (!await continueIfCurrent())
234607
+ return;
234608
+ await dependencies.enqueueFinalize(job2);
234609
+ }
234610
+ async function processGraphStageFailure(job2, error53, dependencies) {
234611
+ if (isStaleRevisionError(error53)) {
234612
+ await dependencies.setGraphStatus(job2.generationId, "cancelled", "stale_revision");
234613
+ await dependencies.cancelStaleRun(job2);
234614
+ return;
234615
+ }
234616
+ if (await dependencies.isCancelled?.(job2))
234617
+ throw error53;
234618
+ await dependencies.setGraphStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "graph_failed");
234619
+ if (!await dependencies.isCancelled?.(job2)) {
234620
+ await dependencies.enqueueSummarize(job2);
234621
+ }
234622
+ throw error53;
234623
+ }
234624
+
234625
+ // ../../backend/src/queue/workers/finalize.worker.ts
233959
234626
  function createFinalizeWorker(deps) {
233960
234627
  return async function processFinalizeJob(input) {
233961
234628
  const job2 = finalizeJobSchema.parse(input);
@@ -233987,6 +234654,7 @@ function createGraphWorker(deps) {
233987
234654
  }
233988
234655
  if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
233989
234656
  await deps.setGraphStatus(job2.generationId, "cancelled", "stale_revision");
234657
+ await deps.cancelStaleRun(job2);
233990
234658
  return;
233991
234659
  }
233992
234660
  if (run4.embedStatus !== "ready") {
@@ -234012,12 +234680,7 @@ function createGraphWorker(deps) {
234012
234680
  if (!await deps.isCancelled?.(job2))
234013
234681
  await deps.enqueueSummarize(job2);
234014
234682
  } catch (error53) {
234015
- if (await deps.isCancelled?.(job2))
234016
- throw error53;
234017
- await deps.setGraphStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "graph_failed");
234018
- if (!await deps.isCancelled?.(job2))
234019
- await deps.enqueueSummarize(job2);
234020
- throw error53;
234683
+ await processGraphStageFailure(job2, error53, deps);
234021
234684
  }
234022
234685
  };
234023
234686
  }
@@ -234088,44 +234751,7 @@ function createPrepareWorker(redisUrl, deps, options = {}) {
234088
234751
  function createSummarizeWorker(deps) {
234089
234752
  return async function processSummarizeJob(input) {
234090
234753
  const job2 = summarizeJobSchema.parse(input);
234091
- if (await deps.isCancelled?.(job2))
234092
- return;
234093
- const run4 = await deps.getRun(job2);
234094
- if (!run4)
234095
- throw new Error("Pipeline run not found");
234096
- if (run4.ownerId !== job2.ownerId || run4.documentId !== job2.documentId) {
234097
- throw new Error("Pipeline owner mismatch");
234098
- }
234099
- if (run4.generationId !== job2.generationId || run4.revision !== job2.revision) {
234100
- await deps.setSummaryStatus(job2.generationId, "cancelled", "stale_revision");
234101
- return;
234102
- }
234103
- if (!deps.enabled()) {
234104
- if (await deps.isCancelled?.(job2))
234105
- return;
234106
- await deps.setSummaryStatus(job2.generationId, "skipped");
234107
- if (!await deps.isCancelled?.(job2))
234108
- await deps.enqueueFinalize(job2);
234109
- return;
234110
- }
234111
- if (await deps.isCancelled?.(job2))
234112
- return;
234113
- await deps.setSummaryStatus(job2.generationId, "processing");
234114
- try {
234115
- if (await deps.isCancelled?.(job2))
234116
- return;
234117
- await deps.summarize(job2);
234118
- if (await deps.isCancelled?.(job2))
234119
- return;
234120
- await deps.setSummaryStatus(job2.generationId, "ready");
234121
- } catch (error53) {
234122
- if (await deps.isCancelled?.(job2))
234123
- return;
234124
- await deps.setSummaryStatus(job2.generationId, "failed", error53 instanceof Error ? error53.name : "summary_failed");
234125
- throw error53;
234126
- }
234127
- if (!await deps.isCancelled?.(job2))
234128
- await deps.enqueueFinalize(job2);
234754
+ await processSummaryStage(job2, deps);
234129
234755
  };
234130
234756
  }
234131
234757
 
@@ -234265,6 +234891,7 @@ var pipelineRuntime = await startRegisteredPipelineWorkers({
234265
234891
  logger3.info({ legacy, recovery }, "Pipeline recovery completed");
234266
234892
  }
234267
234893
  });
234894
+ var reembedCronRuntime = startReembedCron();
234268
234895
  ensureBucket(storage, BUCKET).catch((err) => {
234269
234896
  logger3.error({ err }, "Failed to ensure storage bucket");
234270
234897
  });
@@ -234319,7 +234946,7 @@ var swaggerConfig = {
234319
234946
  },
234320
234947
  info: {
234321
234948
  title: "DocsMint API",
234322
- version: "0.6.0",
234949
+ version: "0.6.2",
234323
234950
  description: "Self-hosted AI-first documentation platform. Full-text + semantic search, version history, sharing, and folder organization.",
234324
234951
  contact: { name: "HiAi-gg", url: "https://github.com/HiAi-gg/docsmint" },
234325
234952
  license: {
@@ -234390,6 +235017,7 @@ app.listen({
234390
235017
  logger3.info({ port: config3.API_PORT }, "hiai-docs API started");
234391
235018
  var stopDocsMintApi = async () => {
234392
235019
  logger3.info("Shutting down...");
235020
+ reembedCronRuntime.close();
234393
235021
  await pipelineRuntime.close();
234394
235022
  await app.stop();
234395
235023
  };