@opengeni/worker-bundle 0.29.0-canary.0 → 0.29.0-canary.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/activities/agent-turn/knowledge-source-tools.d.ts +9 -0
  2. package/dist/activities/agent-turn/knowledge-sources.d.ts +20 -0
  3. package/dist/activities/agent-turn/stream-attempt.d.ts +1 -0
  4. package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
  5. package/dist/activities/knowledge-indexing.d.ts +10 -0
  6. package/dist/activities/run-input.d.ts +1 -0
  7. package/dist/activities/types.d.ts +10 -3
  8. package/dist/{activities-control-MKGEOBXT.js → activities-control-HMFGSKC5.js} +331 -3381
  9. package/dist/activities-control-HMFGSKC5.js.map +1 -0
  10. package/dist/activities-control.d.ts +12 -0
  11. package/dist/{activities-turn-5MZZ6HQA.js → activities-turn-AP67JADS.js} +1296 -1021
  12. package/dist/activities-turn-AP67JADS.js.map +1 -0
  13. package/dist/activities.d.ts +12 -0
  14. package/dist/{chunk-OA5CSY5N.js → chunk-CYITU5UH.js} +3348 -174
  15. package/dist/chunk-CYITU5UH.js.map +1 -0
  16. package/dist/index.d.ts +13 -0
  17. package/dist/index.js +48 -1
  18. package/dist/index.js.map +1 -1
  19. package/dist/workflow-bundle.js +65 -35
  20. package/dist/workflows/knowledge-indexing.d.ts +2 -0
  21. package/dist/workflows.d.ts +1 -0
  22. package/package.json +19 -18
  23. package/src/activities/agent-turn/knowledge-source-tools.ts +98 -0
  24. package/src/activities/agent-turn/knowledge-sources.ts +86 -0
  25. package/src/activities/agent-turn/run.ts +1244 -1112
  26. package/src/activities/agent-turn/skill-install.ts +1 -1
  27. package/src/activities/agent-turn/skill-save.ts +1 -1
  28. package/src/activities/agent-turn/skill-selection.ts +2 -0
  29. package/src/activities/agent-turn/skill-tools.ts +5 -12
  30. package/src/activities/agent-turn/stream-attempt.ts +3 -0
  31. package/src/activities/agent-turn/tool-environment.ts +24 -0
  32. package/src/activities/codex-image-generation.ts +1 -2
  33. package/src/activities/documents.ts +17 -52
  34. package/src/activities/knowledge-indexing.ts +130 -0
  35. package/src/activities/knowledge-source-sync.ts +120 -16
  36. package/src/activities/mcp-credentials.ts +2 -0
  37. package/src/activities/run-input.ts +2 -0
  38. package/src/activities/scheduled-tasks.ts +25 -62
  39. package/src/activities/types.ts +8 -2
  40. package/src/activities/video-generation-reconciliation.ts +8 -1
  41. package/src/activities-control.ts +2 -0
  42. package/src/activities.ts +1 -0
  43. package/src/index.ts +49 -0
  44. package/src/pr-review-credentials.ts +2 -2
  45. package/src/workflows/knowledge-indexing.ts +12 -0
  46. package/src/workflows.ts +1 -0
  47. package/dist/activities-control-MKGEOBXT.js.map +0 -1
  48. package/dist/activities-turn-5MZZ6HQA.js.map +0 -1
  49. package/dist/chunk-OA5CSY5N.js.map +0 -1
@@ -11,7 +11,9 @@ import {
11
11
  assertReusableSessionRevivable,
12
12
  captureConnectedCommandOutput,
13
13
  createCodexCapacityActivities,
14
+ createDocumentActivities,
14
15
  createGoalActivities,
16
+ createKnowledgeSourceSyncActivities,
15
17
  createSharedActivityServices,
16
18
  currentActivityContext,
17
19
  deliverFailedChildTurnToParent,
@@ -31,7 +33,7 @@ import {
31
33
  scheduledUserMessagePayload,
32
34
  withFirstPartyTools,
33
35
  workflowIdForSession
34
- } from "./chunk-OA5CSY5N.js";
36
+ } from "./chunk-CYITU5UH.js";
35
37
  import {
36
38
  inspectOpenSandboxKubernetesInventory,
37
39
  recordCreditBalanceGauges,
@@ -54,6 +56,126 @@ import {
54
56
  } from "./chunk-A4XQWVZK.js";
55
57
  import "./chunk-PZ5AY32C.js";
56
58
 
59
+ // src/activities/knowledge-indexing.ts
60
+ import { configuredStaticUsageLimits } from "@opengeni/config";
61
+ import {
62
+ appendKnowledgeIndexChunks,
63
+ claimKnowledgeIndexJobs,
64
+ completeKnowledgeIndexJob,
65
+ continueKnowledgeIndexJob,
66
+ deferKnowledgeIndexJob,
67
+ getBillingBalance,
68
+ readKnowledgeIndexSource,
69
+ recordUsageEvent,
70
+ sumUsageQuantity,
71
+ withWorkspaceUsageLock
72
+ } from "@opengeni/db";
73
+ function createKnowledgeIndexingActivities(services, resolveDocumentServices) {
74
+ return {
75
+ indexKnowledge: async () => {
76
+ const result = { completed: 0, advanced: 0, deferred: 0, unavailable: 0 };
77
+ if (!resolveDocumentServices) return result;
78
+ const { db, settings, observability } = await services();
79
+ const { embedder } = await resolveDocumentServices();
80
+ const { knowledgeIndexChunks } = await import("@opengeni/documents");
81
+ const claims = await claimKnowledgeIndexJobs(db, {
82
+ model: embedder.model,
83
+ dimensions: embedder.dimensions,
84
+ limit: 2
85
+ });
86
+ for (const claim of claims) {
87
+ try {
88
+ const source = await readKnowledgeIndexSource(db, claim);
89
+ if (!source) {
90
+ result.unavailable++;
91
+ continue;
92
+ }
93
+ await withWorkspaceUsageLock(db, source.billingWorkspaceId, async (lockedDb) => {
94
+ const current = await readKnowledgeIndexSource(lockedDb, claim);
95
+ if (!current) {
96
+ result.unavailable++;
97
+ return;
98
+ }
99
+ const chunks = [];
100
+ let more = false;
101
+ for (const chunk of knowledgeIndexChunks(current.entry)) {
102
+ if (chunk.index < current.nextIndex) continue;
103
+ if (chunks.length === 32) {
104
+ more = true;
105
+ break;
106
+ }
107
+ chunks.push(chunk);
108
+ }
109
+ if (chunks.length) {
110
+ if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
111
+ const balance = await getBillingBalance(lockedDb, claim.accountId);
112
+ if (balance.balanceMicros <= 0) throw new Error("insufficient OpenGeni credits");
113
+ }
114
+ if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
115
+ const limit = configuredStaticUsageLimits(settings).maxDocumentIndexedChunksPerWorkspace;
116
+ if (limit) {
117
+ const now = /* @__PURE__ */ new Date();
118
+ const used = await sumUsageQuantity(lockedDb, {
119
+ workspaceId: current.billingWorkspaceId,
120
+ eventType: "document.indexed",
121
+ since: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1))
122
+ });
123
+ if (used + chunks.length > limit)
124
+ throw new Error("monthly document indexing limit reached");
125
+ }
126
+ }
127
+ const vectors = await embedder.embedMany(chunks.map((chunk) => chunk.embeddingInput));
128
+ if (vectors.length !== chunks.length)
129
+ throw new Error("Incomplete Knowledge embeddings");
130
+ const appended = await appendKnowledgeIndexChunks(
131
+ lockedDb,
132
+ claim,
133
+ current.nextIndex,
134
+ chunks.map((chunk, index) => ({ ...chunk, embedding: vectors[index] }))
135
+ );
136
+ if (appended.status !== "running") {
137
+ result.unavailable++;
138
+ return;
139
+ }
140
+ await recordUsageEvent(lockedDb, {
141
+ accountId: claim.accountId,
142
+ workspaceId: current.billingWorkspaceId,
143
+ eventType: "document.indexed",
144
+ quantity: chunks.length,
145
+ unit: "chunk",
146
+ sourceResourceType: "knowledge_revision",
147
+ sourceResourceId: claim.revisionId,
148
+ idempotencyKey: `knowledge.indexed:${claim.revisionId}:${claim.generation}:${current.nextIndex}`
149
+ });
150
+ }
151
+ if (more) {
152
+ await continueKnowledgeIndexJob(lockedDb, claim);
153
+ result.advanced++;
154
+ } else {
155
+ const completed = await completeKnowledgeIndexJob(
156
+ lockedDb,
157
+ claim,
158
+ current.nextIndex + chunks.length
159
+ );
160
+ if (completed.status === "ready") result.completed++;
161
+ else result.unavailable++;
162
+ }
163
+ });
164
+ } catch {
165
+ await deferKnowledgeIndexJob(db, claim).catch(() => void 0);
166
+ result.deferred++;
167
+ observability.warn("Knowledge indexing batch deferred", {
168
+ accountId: claim.accountId,
169
+ entryId: claim.entryId,
170
+ revisionId: claim.revisionId
171
+ });
172
+ }
173
+ }
174
+ return result;
175
+ }
176
+ };
177
+ }
178
+
57
179
  // src/activities/browser-state-artifact-reaper.ts
58
180
  import {
59
181
  claimBrowserStateArtifactCleanup,
@@ -161,9 +283,9 @@ import {
161
283
  import { SessionSkills } from "@opengeni/contracts";
162
284
 
163
285
  // src/activities/agent-run-admission.ts
164
- import { configuredStaticUsageLimits } from "@opengeni/config";
286
+ import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
165
287
  import { modelFundingForAdmission } from "@opengeni/core";
166
- import { getBillingBalance, isCodexBilledTurn, sumUsageQuantity } from "@opengeni/db";
288
+ import { getBillingBalance as getBillingBalance2, isCodexBilledTurn, sumUsageQuantity as sumUsageQuantity2 } from "@opengeni/db";
167
289
  async function agentRunAdmissionDenial(services, input) {
168
290
  const codexBilled = await isCodexBilledTurn({
169
291
  db: services.db,
@@ -186,16 +308,16 @@ async function agentRunAdmissionDenial(services, input) {
186
308
  });
187
309
  if (!decision.allowed) return "insufficient_credits";
188
310
  } else {
189
- const balance = await getBillingBalance(services.db, input.accountId);
311
+ const balance = await getBillingBalance2(services.db, input.accountId);
190
312
  if (balance.balanceMicros <= 0) return "insufficient_credits";
191
313
  }
192
314
  }
193
315
  if (services.settings.usageLimitsMode !== "static" && services.settings.usageLimitsMode !== "managed") {
194
316
  return null;
195
317
  }
196
- const limits = configuredStaticUsageLimits(services.settings);
318
+ const limits = configuredStaticUsageLimits2(services.settings);
197
319
  if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
198
- const used = await sumUsageQuantity(services.db, {
320
+ const used = await sumUsageQuantity2(services.db, {
199
321
  accountId: input.accountId,
200
322
  eventType: "model.cost",
201
323
  since: startOfUtcMonth()
@@ -205,7 +327,7 @@ async function agentRunAdmissionDenial(services, input) {
205
327
  }
206
328
  }
207
329
  if (limits.maxMonthlyAgentRunsPerWorkspace) {
208
- const used = await sumUsageQuantity(services.db, {
330
+ const used = await sumUsageQuantity2(services.db, {
209
331
  workspaceId: input.workspaceId,
210
332
  eventType: "agent_run.created",
211
333
  since: startOfUtcMonth()
@@ -421,3307 +543,178 @@ function createAutomationActivities(services, options = {}) {
421
543
  }
422
544
  throw error;
423
545
  }
424
- }
425
- };
426
- }
427
- function isUnprocessableEntity(error) {
428
- return error instanceof Error && "status" in error && error.status === 422;
429
- }
430
-
431
- // src/activities/documents.ts
432
- import { configuredStaticUsageLimits as configuredStaticUsageLimits2 } from "@opengeni/config";
433
- import {
434
- getBillingBalance as getBillingBalance2,
435
- recordUsageEvent,
436
- resolveDocumentIndexAuthority,
437
- rlsContextForWorkspace,
438
- sumUsageQuantity as sumUsageQuantity2,
439
- withWorkspaceUsageLock
440
- } from "@opengeni/db";
441
- function createDocumentActivities(services, resolveDocumentServices) {
442
- return {
443
- indexDocument: async (input) => {
444
- const { settings, db, objectStorage } = await services();
445
- if (!objectStorage) {
446
- throw new Error("object storage is not configured");
447
- }
448
- if (!resolveDocumentServices) {
449
- throw new Error("document services are not configured");
450
- }
451
- const [{ getDocumentForIndexing, indexDocumentNow }, documentServices] = await Promise.all([
452
- import("@opengeni/documents"),
453
- resolveDocumentServices()
454
- ]);
455
- const context = await rlsContextForWorkspace(db, input.workspaceId);
456
- if (context.accountId !== input.accountId) {
457
- throw new Error("document account/workspace authority mismatch");
458
- }
459
- return await withWorkspaceUsageLock(db, input.workspaceId, async (lockedDb) => {
460
- const storedAuthority = await resolveDocumentIndexAuthority(lockedDb, input);
461
- if (!storedAuthority) {
462
- throw new Error("document authority was not found before indexing");
463
- }
464
- const suppliedAuthorityFields = [
465
- "authorityKind",
466
- "authorityWorkspaceId",
467
- "authoritySubjectId"
468
- ];
469
- const suppliedCount = suppliedAuthorityFields.filter(
470
- (field) => Object.prototype.hasOwnProperty.call(input, field)
471
- ).length;
472
- const suppliedMismatch = Object.prototype.hasOwnProperty.call(input, "authorityKind") && input.authorityKind !== storedAuthority.authorityKind || Object.prototype.hasOwnProperty.call(input, "authorityWorkspaceId") && input.authorityWorkspaceId !== storedAuthority.authorityWorkspaceId || Object.prototype.hasOwnProperty.call(input, "authoritySubjectId") && input.authoritySubjectId !== storedAuthority.authoritySubjectId;
473
- if (suppliedMismatch) {
474
- throw new Error("document authority changed before indexing");
475
- }
476
- if (suppliedCount !== 0 && suppliedCount !== suppliedAuthorityFields.length) {
477
- throw new Error("document authority tuple is partial");
478
- }
479
- const claimedDocument = await getDocumentForIndexing(
480
- lockedDb,
481
- input.workspaceId,
482
- input.documentId
483
- );
484
- if (!claimedDocument || claimedDocument.authorityKind !== storedAuthority.authorityKind || claimedDocument.authorityWorkspaceId !== storedAuthority.authorityWorkspaceId || claimedDocument.authoritySubjectId !== storedAuthority.authoritySubjectId) {
485
- throw new Error("document authority changed before indexing");
486
- }
487
- const document = await indexDocumentNow(
488
- lockedDb,
489
- objectStorage,
490
- input.workspaceId,
491
- input.documentId,
492
- documentServices,
493
- {
494
- beforeEmbed: async ({ chunkCount }) => {
495
- if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
496
- const balance = await getBillingBalance2(lockedDb, input.accountId);
497
- if (balance.balanceMicros <= 0) {
498
- throw new Error("insufficient OpenGeni credits");
499
- }
500
- }
501
- if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
502
- return;
503
- }
504
- const limit = configuredStaticUsageLimits2(settings).maxDocumentIndexedChunksPerWorkspace;
505
- if (!limit) {
506
- return;
507
- }
508
- const used = await sumUsageQuantity2(lockedDb, {
509
- workspaceId: input.workspaceId,
510
- eventType: "document.indexed",
511
- since: startOfUtcMonth2()
512
- });
513
- if (used + chunkCount > limit) {
514
- throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
515
- }
516
- }
517
- },
518
- { viewerSubjectId: storedAuthority.authoritySubjectId }
519
- );
520
- if (document.authorityKind !== storedAuthority.authorityKind || document.authorityWorkspaceId !== storedAuthority.authorityWorkspaceId || document.authoritySubjectId !== storedAuthority.authoritySubjectId) {
521
- throw new Error("document authority changed before indexing");
522
- }
523
- if (document.status === "ready") {
524
- await recordUsageEvent(lockedDb, {
525
- accountId: input.accountId,
526
- workspaceId: input.workspaceId,
527
- eventType: "document.indexed",
528
- quantity: document.chunkCount,
529
- unit: "chunk",
530
- sourceResourceType: "document",
531
- sourceResourceId: document.id,
532
- idempotencyKey: `document.indexed:${input.workspaceId}:${document.id}:${document.updatedAt}`
533
- });
534
- }
535
- return document;
536
- });
537
- }
538
- };
539
- }
540
- function startOfUtcMonth2(date = /* @__PURE__ */ new Date()) {
541
- return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
542
- }
543
-
544
- // src/activities/file-upload-reaper.ts
545
- import {
546
- claimDueTranscriptionRecordingObjectCleanup,
547
- claimExpiredFileUploadCleanup,
548
- completeDueTranscriptionRecordingObjectCleanup,
549
- completeExpiredFileUploadCleanup,
550
- claimVideoGenerationOperations,
551
- purgeExpiredTranscriptionRecordings
552
- } from "@opengeni/db";
553
- import { randomUUID } from "crypto";
554
- var FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1e3;
555
- var FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
556
- var FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
557
- var SAFE_CLEANUP_ERROR_CATEGORIES = /* @__PURE__ */ new Set([
558
- "aborted",
559
- "conflict",
560
- "invalid_response",
561
- "network",
562
- "not_found",
563
- "permission_denied",
564
- "provider",
565
- "timeout",
566
- "unavailable"
567
- ]);
568
- function cleanupErrorAttributes(error) {
569
- let errorCategory = "provider_error";
570
- let errorStatus;
571
- if (error && typeof error === "object") {
572
- const rawCode = error.code;
573
- if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
574
- errorCategory = rawCode;
575
- } else if (rawCode !== void 0) {
576
- errorCategory = "unknown";
577
- }
578
- const rawStatus = Number(
579
- error.status ?? error.statusCode
580
- );
581
- if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
582
- errorStatus = rawStatus;
583
- }
584
- }
585
- return { errorCategory, ...errorStatus === void 0 ? {} : { errorStatus } };
586
- }
587
- function createFileUploadReaperActivities(services, options = {}) {
588
- const graceMs = options.graceMs ?? FILE_UPLOAD_CLEANUP_GRACE_MS;
589
- const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
590
- const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
591
- const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
592
- const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
593
- const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
594
- const claimTranscriptionObjects = options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
595
- const completeTranscriptionObject = options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
596
- const purgeTranscriptionRecordings = options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
597
- async function reapExpiredFileUploads() {
598
- const { db, objectStorage, observability } = await services();
599
- if (!objectStorage) {
600
- try {
601
- await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
602
- } catch (error) {
603
- observability.warn("expired transcription recording metadata purge failed", {
604
- ...cleanupErrorAttributes(error)
605
- });
606
- }
607
- return { claimed: 0, deleted: 0, failed: 0 };
608
- }
609
- const claims = await claimFileUploads(db, {
610
- graceMs,
611
- claimTimeoutMs,
612
- limit: batchSize
613
- });
614
- let deleted = 0;
615
- let failed = 0;
616
- for (const claim of claims) {
617
- try {
618
- await deleteObject(objectStorage, claim.objectKey);
619
- const settled = await completeFileUpload(db, claim);
620
- if (!settled) {
621
- throw new Error("cleanup claim no longer owns a reclaimable upload");
622
- }
623
- deleted += 1;
624
- } catch (error) {
625
- failed += 1;
626
- observability.warn("expired file upload cleanup failed; claim remains reclaimable", {
627
- workspaceId: claim.workspaceId,
628
- uploadId: claim.uploadId,
629
- fileId: claim.fileId,
630
- ...cleanupErrorAttributes(error)
631
- });
632
- }
633
- }
634
- const recordingClaims = await claimTranscriptionObjects(db, {
635
- graceMs,
636
- claimTimeoutMs,
637
- limit: batchSize
638
- });
639
- for (const claim of recordingClaims) {
640
- try {
641
- await deleteObject(objectStorage, claim.objectKey);
642
- const settled = await completeTranscriptionObject(db, claim);
643
- if (!settled) {
644
- throw new Error("cleanup claim no longer owns a transcription recording object");
645
- }
646
- deleted += 1;
647
- } catch (error) {
648
- failed += 1;
649
- observability.warn(
650
- "expired transcription recording object cleanup failed; claim remains reclaimable",
651
- {
652
- workspaceId: claim.workspaceId,
653
- recordingId: claim.recordingId,
654
- ...cleanupErrorAttributes(error)
655
- }
656
- );
657
- }
658
- }
659
- let transcriptionRecordingsPurged = 0;
660
- try {
661
- transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
662
- graceMs,
663
- limit: batchSize
664
- });
665
- } catch (error) {
666
- observability.warn("expired transcription recording metadata purge failed", {
667
- ...cleanupErrorAttributes(error)
668
- });
669
- }
670
- const claimed = claims.length + recordingClaims.length;
671
- if (claimed > 0 || transcriptionRecordingsPurged > 0) {
672
- observability.info("expired file upload cleanup swept", {
673
- claimed,
674
- deleted,
675
- failed,
676
- fileUploads: claims.length,
677
- transcriptionRecordingObjects: recordingClaims.length,
678
- transcriptionRecordingsPurged
679
- });
680
- }
681
- return { claimed, deleted, failed };
682
- }
683
- async function recoverVideoGenerationWorkflows() {
684
- const { db, startVideoGenerationWorkflow, observability } = await services();
685
- if (!startVideoGenerationWorkflow) return { claimed: 0, started: 0, failed: 0 };
686
- const claims = await claimVideoGenerationOperations(db, {
687
- owner: `video-workflow-recovery:${randomUUID()}`,
688
- leaseSeconds: 120,
689
- limit: 100
690
- });
691
- let started = 0;
692
- let failed = 0;
693
- for (const claim of claims) {
694
- try {
695
- await startVideoGenerationWorkflow(claim);
696
- started += 1;
697
- } catch (error) {
698
- failed += 1;
699
- observability.warn("Video generation workflow recovery start failed", {
700
- operationId: claim.operationId,
701
- errorClass: error instanceof Error ? error.name : "UnknownError"
702
- });
703
- }
704
- }
705
- return { claimed: claims.length, started, failed };
706
- }
707
- return { reapExpiredFileUploads, recoverVideoGenerationWorkflows };
708
- }
709
-
710
- // src/activities/knowledge-source-sync.ts
711
- import { createHash as createHash2 } from "crypto";
712
- import { heartbeat } from "@temporalio/activity";
713
- import {
714
- googleDriveProviderRetryOptions
715
- } from "@opengeni/config";
716
- import {
717
- KnowledgeSourceSyncRunSummary
718
- } from "@opengeni/contracts";
719
- import {
720
- ATLASSIAN_PROVIDER_DOMAIN,
721
- AtlassianConnectionMetadata,
722
- atlassianScopesAllowRead
723
- } from "@opengeni/contracts/atlassian";
724
- import {
725
- GoogleDriveConnectionMetadata,
726
- GOOGLE_DRIVE_PROVIDER_DOMAIN,
727
- googleDriveScopesAllowCapability
728
- } from "@opengeni/contracts/google-drive";
729
- import {
730
- googleDriveKnowledgeScope,
731
- inventoryGoogleDriveSource
732
- } from "@opengeni/documents/google-drive";
733
- import {
734
- atlassianKnowledgeScope,
735
- inventoryAtlassianSource
736
- } from "@opengeni/documents/atlassian";
737
- import { addDocumentToBase, ensureDefaultBase, getDocumentBase } from "@opengeni/documents";
738
- import {
739
- appendKnowledgeDocumentVersion,
740
- appendKnowledgeSourceAclVersion,
741
- beginKnowledgeSyncRun,
742
- beginGoogleDriveObjectAclRefresh,
743
- buildConnectionTokenResolver,
744
- checkpointKnowledgeSourceSync,
745
- claimKnowledgeSourceSyncLease,
746
- compareCanonicalDecimalProviderRevisions,
747
- completeKnowledgeSyncRun,
748
- completeKnowledgeSourceSyncWakeWithoutLease,
749
- deauthorizeKnowledgeSourceRetrieval,
750
- enqueueKnowledgeSourceSyncIndexObligation,
751
- ensureKnowledgeSourceBlobFile,
752
- getConnectionMetadata,
753
- getKnowledgeDocumentVersionObservationForSyncAuthority,
754
- getKnowledgeSyncRunForSyncAuthority,
755
- getKnowledgeSourceAclForSyncAuthority,
756
- getKnowledgeSourceForSyncAuthority,
757
- getKnowledgeSourceObjectForSyncAuthority,
758
- getKnowledgeSourceSyncIndexObligationForVersion,
759
- getScheduledTask,
760
- listObservedKnowledgeSourceSyncExternalObjectIds,
761
- KnowledgeSourceSyncObservationFenceError,
762
- reconcileKnowledgeSourceSyncCompleteScan,
763
- reconcileKnowledgeSourceSyncLiveGeneration,
764
- recordGoogleDriveObjectAclEvidence,
765
- recordKnowledgeSourceSyncItemOutcomes,
766
- recordKnowledgeSourceSyncObjectObservations,
767
- recordUsageEvent as recordUsageEvent2,
768
- releaseKnowledgeSourceSyncLeaseForRetry,
769
- restoreKnowledgeSourceObject,
770
- retryKnowledgeSourceSyncIndexObligation,
771
- scopedKnowledgeScopeKey,
772
- settleKnowledgeSourceSyncLease,
773
- settleKnowledgeSourceSyncIndexObligation,
774
- updateKnowledgeSourceDocumentObservationMetadata,
775
- updateScheduledTaskRun,
776
- upsertKnowledgeSourceObject
777
- } from "@opengeni/db";
778
- import { readResponseJsonBounded } from "@opengeni/network";
779
-
780
- // src/activities/google-drive-provider.ts
781
- var GoogleDriveProviderTransportError = class extends Error {
782
- constructor() {
783
- super("google_drive_provider_transport_failed");
784
- this.name = "GoogleDriveProviderTransportError";
785
- }
786
- };
787
- async function fetchGoogleDriveProvider(input) {
788
- const now = input.now ?? Date.now;
789
- const sleep = input.sleep ?? Bun.sleep;
790
- let retryDelaySpentMs = 0;
791
- for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
792
- let response;
793
- input.heartbeat?.({ operation: input.operation, attempt, phase: "request_start" });
794
- try {
795
- response = await input.fetchImpl(input.url, {
796
- ...input.init,
797
- redirect: "error",
798
- signal: AbortSignal.timeout(input.policy.requestTimeoutMs)
799
- });
800
- } catch {
801
- input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
802
- recordRequest(input.observability, input.operation, "retryable_error");
803
- const delayMs2 = retryDelayMs(input.policy, attempt, null);
804
- if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs2)) {
805
- throw new GoogleDriveProviderTransportError();
806
- }
807
- recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
808
- retryDelaySpentMs += delayMs2;
809
- input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
810
- await sleep(delayMs2);
811
- input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
812
- continue;
813
- }
814
- input.heartbeat?.({ operation: input.operation, attempt, phase: "request_end" });
815
- if (!isRetryableStatus(response.status)) {
816
- recordRequest(
817
- input.observability,
818
- input.operation,
819
- response.ok ? "succeeded" : "terminal_error"
820
- );
821
- return response;
822
- }
823
- recordRequest(input.observability, input.operation, "retryable_error");
824
- const delayMs = retryDelayMs(
825
- input.policy,
826
- attempt,
827
- retryAfterDelayMs(response.headers.get("retry-after"), now())
828
- );
829
- if (!canRetry(input.policy, attempt, retryDelaySpentMs, delayMs)) return response;
830
- await response.body?.cancel().catch(() => void 0);
831
- recordRetry(
832
- input.observability,
833
- input.operation,
834
- response.status === 429 ? "rate_limited" : "provider_unavailable",
835
- attempt,
836
- delayMs
837
- );
838
- retryDelaySpentMs += delayMs;
839
- input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
840
- await sleep(delayMs);
841
- input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
842
- }
843
- throw new GoogleDriveProviderTransportError();
844
- }
845
- function canRetry(policy, attempt, retryDelaySpentMs, delayMs) {
846
- return attempt < policy.attempts && retryDelaySpentMs + delayMs <= policy.budgetMs;
847
- }
848
- function retryDelayMs(policy, attempt, retryAfterMs) {
849
- const exponential = Math.min(
850
- policy.maxDelayMs,
851
- policy.initialDelayMs * 2 ** Math.max(0, attempt - 1)
852
- );
853
- return Math.min(policy.maxDelayMs, Math.max(exponential, retryAfterMs ?? 0));
854
- }
855
- function retryAfterDelayMs(value, nowMs) {
856
- if (!value) return null;
857
- const seconds = Number(value);
858
- if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1e3);
859
- const timestamp = Date.parse(value);
860
- return Number.isFinite(timestamp) ? Math.max(0, timestamp - nowMs) : null;
861
- }
862
- function isRetryableStatus(status) {
863
- return status === 429 || status >= 500;
864
- }
865
- function recordRequest(observability, operation, outcome) {
866
- observability.incrementCounter({
867
- name: "opengeni_google_drive_provider_requests_total",
868
- help: "Google Drive provider request attempts by operation and bounded outcome.",
869
- labels: { operation, outcome }
870
- });
871
- }
872
- function recordRetry(observability, operation, reason, attempt, delayMs) {
873
- observability.incrementCounter({
874
- name: "opengeni_google_drive_provider_retries_total",
875
- help: "Bounded Google Drive provider retries by operation and reason.",
876
- labels: { operation, reason }
877
- });
878
- observability.observeHistogram({
879
- name: "opengeni_google_drive_provider_retry_delay_seconds",
880
- help: "Delay applied before a bounded Google Drive provider retry.",
881
- labels: { operation, reason },
882
- value: delayMs / 1e3,
883
- buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]
884
- });
885
- observability.warn("Google Drive provider request retry scheduled", {
886
- provider: "google_drive",
887
- operation,
888
- outcome: "retry_scheduled",
889
- reason,
890
- attempt,
891
- delayMs
892
- });
893
- }
894
-
895
- // src/activities/google-drive-changes.ts
896
- import { createHash } from "crypto";
897
- import {
898
- GOOGLE_DRIVE_FOLDER_MIME_TYPE,
899
- planGoogleDriveTransfer
900
- } from "@opengeni/documents/google-drive";
901
- var MAX_TOKEN_CHARS = 4096;
902
- var MAX_BOUNDARY_ID_CHARS = 1024;
903
- var MAX_ANCESTOR_DEPTH = 100;
904
- function parseGoogleDriveChangesCursor(value, expected) {
905
- if (!value || value.version !== 1 || value.kind !== "google_drive_changes") return null;
906
- if (value.connectionId !== expected.connectionId || value.googlePermissionId !== expected.googlePermissionId || value.sourceId !== expected.sourceId || value.driveId !== expected.driveId || typeof value.boundaryId !== "string" || value.boundaryId.length < 1 || value.boundaryId.length > MAX_BOUNDARY_ID_CHARS || typeof value.pageToken !== "string" || value.pageToken.length < 1 || value.pageToken.length > MAX_TOKEN_CHARS || typeof value.cursorGeneration !== "number" || !Number.isSafeInteger(value.cursorGeneration) || value.cursorGeneration < 1 || typeof value.lastFullReconciliationAt !== "string" || !Number.isFinite(Date.parse(value.lastFullReconciliationAt)) || typeof value.nextFullReconciliationAt !== "string" || !Number.isFinite(Date.parse(value.nextFullReconciliationAt))) {
907
- return null;
908
- }
909
- return value;
910
- }
911
- function buildGoogleDriveChangesCursor(input) {
912
- return {
913
- version: 1,
914
- kind: "google_drive_changes",
915
- connectionId: input.connectionId,
916
- googlePermissionId: input.googlePermissionId,
917
- sourceId: input.sourceId,
918
- driveId: input.driveId,
919
- boundaryId: boundedBoundaryId(input.boundaryId),
920
- pageToken: boundedToken(input.pageToken),
921
- cursorGeneration: input.previousGeneration + 1,
922
- lastFullReconciliationAt: input.reconciledAt.toISOString(),
923
- nextFullReconciliationAt: new Date(
924
- input.reconciledAt.getTime() + input.fullReconciliationIntervalMs
925
- ).toISOString()
926
- };
927
- }
928
- function advanceGoogleDriveChangesCursor(cursor, pageToken) {
929
- return {
930
- ...cursor,
931
- pageToken: boundedToken(pageToken),
932
- cursorGeneration: cursor.cursorGeneration + 1
933
- };
934
- }
935
- function googleDriveFullReconciliationDue(cursor, now) {
936
- return Date.parse(cursor.nextFullReconciliationAt) <= now.getTime();
937
- }
938
- async function resolveGoogleDriveChangesBoundaryId(input) {
939
- if (input.source.id !== "root" || input.source.driveId !== null) {
940
- return boundedBoundaryId(input.source.id);
941
- }
942
- const root = await input.getFile("root");
943
- if (!root || root.mimeType !== GOOGLE_DRIVE_FOLDER_MIME_TYPE || root.trashed) {
944
- throw new GoogleDriveChangesProtocolError("google_drive_root_boundary_invalid");
945
- }
946
- return boundedBoundaryId(root.id);
947
- }
948
- function parseGoogleDriveChangesCheckpoint(value, limits) {
949
- try {
950
- return validatedCheckpoint(value, limits);
951
- } catch {
952
- return null;
953
- }
954
- }
955
- async function drainGoogleDriveChanges(input) {
956
- const now = input.now ?? Date.now;
957
- const startedAt = now();
958
- const checkpoint = input.checkpoint ? validatedCheckpoint(input.checkpoint, input) : null;
959
- let pageToken = boundedToken(checkpoint?.pageToken ?? input.cursor.pageToken);
960
- let requiresFullReconciliation = checkpoint?.requiresFullReconciliation ?? false;
961
- const seenPageTokenHashes = new Set(checkpoint?.seenPageTokenHashes ?? []);
962
- const initialBudget = checkpoint?.budget ?? emptyBudget();
963
- const budget = { ...initialBudget };
964
- const entries = /* @__PURE__ */ new Map();
965
- while (true) {
966
- const invocationElapsedMs = Math.max(0, now() - startedAt);
967
- budget.elapsedMs = initialBudget.elapsedMs + invocationElapsedMs;
968
- const stopReason = limitReason(budget, input, invocationElapsedMs);
969
- if (stopReason) {
970
- return {
971
- status: "paused",
972
- stopReason,
973
- entries: [...entries.values()],
974
- checkpoint: {
975
- version: 2,
976
- pageToken,
977
- requiresFullReconciliation,
978
- seenPageTokenHashes: [...seenPageTokenHashes],
979
- budget: { ...budget }
980
- },
981
- newStartPageToken: null,
982
- requiresFullReconciliation,
983
- providerRequests: budget.providerRequests,
984
- elapsedMs: budget.elapsedMs,
985
- budget: { ...budget },
986
- hardLimitReached: stopReason !== "elapsed_time_limit" || budget.elapsedMs >= input.maxElapsedMs
987
- };
988
- }
989
- const pageTokenHash = createHash("sha256").update(pageToken).digest("hex");
990
- if (seenPageTokenHashes.has(pageTokenHash)) {
991
- throw new GoogleDriveChangesProtocolError("google_drive_changes_page_token_cycle");
992
- }
993
- seenPageTokenHashes.add(pageTokenHash);
994
- budget.providerRequests += 1;
995
- const pageSize = Math.min(100, input.maxItems - budget.examinedItems);
996
- let page;
997
- try {
998
- page = await input.listChanges(pageToken, pageSize);
999
- } catch (error) {
1000
- budget.elapsedMs = initialBudget.elapsedMs + Math.max(0, now() - startedAt);
1001
- if (error instanceof GoogleDriveCursorInvalidError) error.budget = { ...budget };
1002
- throw error;
1003
- }
1004
- if (page.changes.length > pageSize) {
1005
- throw new GoogleDriveChangesProtocolError("google_drive_changes_page_exceeds_item_budget");
1006
- }
1007
- budget.examinedItems += page.changes.length;
1008
- const repairCandidates = /* @__PURE__ */ new Set();
1009
- for (const change of page.changes) {
1010
- if (change.removed || !change.file || change.file.trashed) {
1011
- repairCandidates.add(change.fileId);
1012
- continue;
1013
- }
1014
- const membership = await isInsideSource({
1015
- file: change.file,
1016
- boundaryId: input.cursor.boundaryId,
1017
- getFile: async (fileId) => {
1018
- const ancestryElapsedMs = Math.max(0, now() - startedAt);
1019
- budget.elapsedMs = initialBudget.elapsedMs + ancestryElapsedMs;
1020
- if (budget.providerRequests >= input.maxProviderRequests || budget.elapsedMs >= input.maxElapsedMs || ancestryElapsedMs >= input.maxInvocationElapsedMs) {
1021
- return null;
1022
- }
1023
- budget.providerRequests += 1;
1024
- const parent = await input.getFile(fileId);
1025
- budget.elapsedMs = initialBudget.elapsedMs + Math.max(0, now() - startedAt);
1026
- return parent;
1027
- }
1028
- });
1029
- if (change.file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
1030
- entries.delete(change.fileId);
1031
- if (membership !== "outside") requiresFullReconciliation = true;
1032
- else repairCandidates.add(change.fileId);
1033
- continue;
1034
- }
1035
- if (membership === "inside") {
1036
- entries.set(change.fileId, toInventoryEntry(change.file, input.source, input.maxFileBytes));
1037
- } else if (membership === "unknown") {
1038
- entries.delete(change.fileId);
1039
- requiresFullReconciliation = true;
1040
- } else {
1041
- repairCandidates.add(change.fileId);
1042
- }
1043
- }
1044
- if (repairCandidates.size > 0) {
1045
- for (const fileId of repairCandidates) {
1046
- if (entries.delete(fileId)) requiresFullReconciliation = true;
1047
- }
1048
- const observed = await input.observedExternalObjectIds([...repairCandidates]);
1049
- if (observed.size > 0) requiresFullReconciliation = true;
1050
- }
1051
- if (page.nextPageToken) {
1052
- pageToken = boundedToken(page.nextPageToken);
1053
- continue;
1054
- }
1055
- if (!page.newStartPageToken) {
1056
- throw new GoogleDriveChangesProtocolError("google_drive_changes_terminal_token_missing");
1057
- }
1058
- budget.elapsedMs = initialBudget.elapsedMs + Math.max(0, now() - startedAt);
1059
- return {
1060
- status: "complete",
1061
- stopReason: null,
1062
- entries: [...entries.values()],
1063
- checkpoint: null,
1064
- newStartPageToken: boundedToken(page.newStartPageToken),
1065
- requiresFullReconciliation,
1066
- providerRequests: budget.providerRequests,
1067
- elapsedMs: budget.elapsedMs,
1068
- budget: { ...budget },
1069
- hardLimitReached: false
1070
- };
1071
- }
1072
- }
1073
- async function isInsideSource(input) {
1074
- if (input.file.id === input.boundaryId) return "inside";
1075
- let parents = input.file.parents;
1076
- const seen = /* @__PURE__ */ new Set([input.file.id]);
1077
- for (let depth = 0; depth < MAX_ANCESTOR_DEPTH; depth += 1) {
1078
- if (parents.includes(input.boundaryId)) return "inside";
1079
- const next = parents.find((parent2) => !seen.has(parent2));
1080
- if (!next) return "outside";
1081
- seen.add(next);
1082
- const parent = await input.getFile(next);
1083
- if (!parent) return "unknown";
1084
- parents = parent.parents;
1085
- }
1086
- return "unknown";
1087
- }
1088
- function toInventoryEntry(file, source, maxFileBytes) {
1089
- return {
1090
- externalObjectId: file.id,
1091
- externalVersionId: file.version ?? file.modifiedTime ?? file.md5Checksum,
1092
- sourceId: source.id,
1093
- parentFolderId: file.parents[0] ?? source.id,
1094
- driveId: file.driveId ?? source.driveId,
1095
- title: file.name,
1096
- mimeType: file.mimeType,
1097
- modifiedTime: file.modifiedTime,
1098
- createdTime: file.createdTime,
1099
- sourceUri: file.webViewLink ?? `https://drive.google.com/open?id=${encodeURIComponent(file.id)}`,
1100
- transfer: planGoogleDriveTransfer(file, maxFileBytes)
1101
- };
1102
- }
1103
- function validatedCheckpoint(value, limits) {
1104
- if (!value || typeof value !== "object") {
1105
- throw new Error("google_drive_changes_checkpoint_invalid");
1106
- }
1107
- const checkpoint = value;
1108
- const budget = checkpoint.budget;
1109
- if (checkpoint.version !== 2 || typeof checkpoint.requiresFullReconciliation !== "boolean" || !Array.isArray(checkpoint.seenPageTokenHashes) || checkpoint.seenPageTokenHashes.some((hash) => !/^[a-f0-9]{64}$/.test(hash)) || new Set(checkpoint.seenPageTokenHashes).size !== checkpoint.seenPageTokenHashes.length || checkpoint.seenPageTokenHashes.length > limits.maxProviderRequests || !budget || typeof budget !== "object") {
1110
- throw new Error("google_drive_changes_checkpoint_invalid");
1111
- }
1112
- if (typeof checkpoint.pageToken !== "string") {
1113
- throw new Error("google_drive_changes_checkpoint_invalid");
1114
- }
1115
- boundedToken(checkpoint.pageToken);
1116
- const budgetRow = budget;
1117
- for (const [key, maximum] of [
1118
- ["examinedItems", limits.maxItems],
1119
- ["providerRequests", limits.maxProviderRequests],
1120
- ["elapsedMs", limits.maxElapsedMs]
1121
- ]) {
1122
- const budgetValue = budgetRow[key];
1123
- if (!Number.isSafeInteger(budgetValue) || Number(budgetValue) < 0 || Number(budgetValue) > maximum) {
1124
- throw new Error("google_drive_changes_checkpoint_budget_invalid");
1125
- }
1126
- }
1127
- if (checkpoint.seenPageTokenHashes.length > Number(budgetRow.providerRequests)) {
1128
- throw new Error("google_drive_changes_checkpoint_request_history_invalid");
1129
- }
1130
- return value;
1131
- }
1132
- function limitReason(budget, input, invocationElapsedMs) {
1133
- if (budget.examinedItems >= input.maxItems) return "item_limit";
1134
- if (budget.providerRequests >= input.maxProviderRequests) return "api_request_limit";
1135
- if (budget.elapsedMs >= input.maxElapsedMs || invocationElapsedMs >= input.maxInvocationElapsedMs) {
1136
- return "elapsed_time_limit";
1137
- }
1138
- return null;
1139
- }
1140
- function emptyBudget() {
1141
- return { examinedItems: 0, providerRequests: 0, elapsedMs: 0 };
1142
- }
1143
- function boundedToken(value) {
1144
- if (value.length < 1 || value.length > MAX_TOKEN_CHARS) {
1145
- throw new Error("google_drive_changes_token_invalid");
1146
- }
1147
- return value;
1148
- }
1149
- function boundedBoundaryId(value) {
1150
- if (value.length < 1 || value.length > MAX_BOUNDARY_ID_CHARS) {
1151
- throw new Error("google_drive_changes_boundary_invalid");
1152
- }
1153
- return value;
1154
- }
1155
- var GoogleDriveCursorInvalidError = class extends Error {
1156
- budget = null;
1157
- constructor() {
1158
- super("google_drive_changes_cursor_invalid");
1159
- this.name = "GoogleDriveCursorInvalidError";
1160
- }
1161
- };
1162
- var GoogleDriveChangesProtocolError = class extends Error {
1163
- constructor(message) {
1164
- super(message);
1165
- this.name = "GoogleDriveChangesProtocolError";
1166
- }
1167
- };
1168
-
1169
- // src/activities/knowledge-source-sync.ts
1170
- var DRIVE_API_BASE = "https://www.googleapis.com/drive/v3";
1171
- var DRIVE_JSON_MAX_BYTES = 2 * 1024 * 1024;
1172
- var GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS = 24 * 60 * 60 * 1e3;
1173
- var GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS = 3e4;
1174
- var GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES = 2 * 1024 * 1024;
1175
- var GOOGLE_DRIVE_ACL_FRESHNESS_MS = 26 * 60 * 60 * 1e3;
1176
- var GOOGLE_DRIVE_ACL_MAX_PRINCIPALS = 1e3;
1177
- var ATLASSIAN_API_BASE = "https://api.atlassian.com";
1178
- var ATLASSIAN_JSON_MAX_BYTES = 4 * 1024 * 1024;
1179
- function knowledgeSyncObservationPolicy(kind) {
1180
- return kind === "google_drive" ? { revisionOrdering: "canonical_decimal", filterWithDriveDurability: true } : { revisionOrdering: "first_observation", filterWithDriveDurability: false };
1181
- }
1182
- function createKnowledgeSourceSyncActivities(services, resolveDocumentServices) {
1183
- const documentActivities = createDocumentActivities(services, resolveDocumentServices);
1184
- return {
1185
- runKnowledgeSourceSyncBatch: async (input) => {
1186
- const { settings, db, objectStorage, observability } = await services();
1187
- if (!objectStorage) throw new Error("object storage is not configured");
1188
- const task = await getScheduledTask(db, input.workspaceId, input.taskId);
1189
- if (!task || task.accountId !== input.accountId || task.action.kind !== "knowledge_source_sync" || task.action.sourceId !== input.sourceId || task.status !== "active") {
1190
- return await failWithoutProvider("schedule_inactive_or_changed");
1191
- }
1192
- const action = task.action;
1193
- const providerLabel = action.connection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN ? "atlassian" : "google_drive";
1194
- if (action.controlWorkspaceId !== input.workspaceId || action.sourceConfigGeneration <= 0 || !action.providerCoordinationKey) {
1195
- return await failWithoutProvider("authority_changed");
1196
- }
1197
- const lease = await claimKnowledgeSourceSyncLease(db, {
1198
- ...input,
1199
- overlapPolicy: input.overlapPolicy
1200
- });
1201
- if (lease.action !== "claimed") {
1202
- observability.incrementCounter({
1203
- name: "opengeni_knowledge_source_sync_runs_total",
1204
- labels: { provider: providerLabel, outcome: lease.action }
1205
- });
1206
- return { action: lease.action };
1207
- }
1208
- let knowledgeRunId = null;
1209
- let activeSourceSyncGeneration = null;
1210
- const startedAt = Date.now();
1211
- const summary = {
1212
- phase: "queued",
1213
- scanned: 0,
1214
- imported: 0,
1215
- unchanged: 0,
1216
- skipped: 0,
1217
- failed: 0,
1218
- bytes: 0,
1219
- providerRequests: 0,
1220
- elapsedMs: 0,
1221
- indexed: 0,
1222
- aclPending: 0,
1223
- retryable: false,
1224
- limitReached: null,
1225
- checkpointed: false,
1226
- reconnectRequired: false,
1227
- failures: []
1228
- };
1229
- observability.info("Knowledge source sync started", {
1230
- provider: providerLabel,
1231
- outcome: "started"
1232
- });
1233
- try {
1234
- summary.phase = "inventory";
1235
- const resolved = await getKnowledgeSourceForSyncAuthority(db, {
1236
- accountId: input.accountId,
1237
- workspaceId: input.workspaceId,
1238
- sourceId: input.sourceId,
1239
- initiatingSubjectId: action.initiatingSubjectId
1240
- });
1241
- if (!resolved || resolved.source.lifecycleState !== "active") {
1242
- throw new SyncFailure("authority_changed", false);
1243
- }
1244
- if (scopedKnowledgeScopeKey(resolved.source.scope) !== scopedKnowledgeScopeKey(action.destination)) {
1245
- throw new SyncFailure("authority_changed", false);
1246
- }
1247
- const liveState = await reconcileKnowledgeSourceSyncLiveGeneration(db, {
1248
- accountId: input.accountId,
1249
- workspaceId: input.workspaceId,
1250
- sourceId: input.sourceId,
1251
- scheduledTaskRunId: input.scheduledTaskRunId,
1252
- initiatingSubjectId: action.initiatingSubjectId,
1253
- sourceConfigGeneration: action.sourceConfigGeneration,
1254
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1255
- liveSourceSyncGeneration: resolved.source.syncGeneration
1256
- });
1257
- activeSourceSyncGeneration = resolved.source.syncGeneration;
1258
- const syncOperationId = `scheduled-task-run:${input.scheduledTaskRunId}`;
1259
- const replayedRun = await getKnowledgeSyncRunForSyncAuthority(db, {
1260
- accountId: input.accountId,
1261
- workspaceId: input.workspaceId,
1262
- sourceId: input.sourceId,
1263
- operationId: syncOperationId,
1264
- initiatingSubjectId: action.initiatingSubjectId
1265
- });
1266
- if (replayedRun && replayedRun.state !== "started") {
1267
- const replayedSummary = KnowledgeSourceSyncRunSummary.parse(replayedRun.metadata);
1268
- const settlementInput = {
1269
- accountId: input.accountId,
1270
- workspaceId: input.workspaceId,
1271
- sourceId: input.sourceId,
1272
- scheduledTaskRunId: input.scheduledTaskRunId,
1273
- knowledgeSyncRunId: replayedRun.id,
1274
- status: replayedRun.state === "succeeded" ? "succeeded" : "failed",
1275
- summary: replayedSummary,
1276
- error: replayedRun.errorCode,
1277
- sourceConfigGeneration: action.sourceConfigGeneration,
1278
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1279
- sourceSyncGeneration: replayedRun.inputSyncGeneration,
1280
- completedSourceSyncGeneration: replayedRun.state === "succeeded" ? replayedRun.inputSyncGeneration + 1 : null,
1281
- ...replayedRun.state === "succeeded" && replayedRun.metadata.providerCursor && typeof replayedRun.metadata.providerCursor === "object" ? {
1282
- providerCursor: replayedRun.metadata.providerCursor
1283
- } : {}
1284
- };
1285
- const settled2 = replayedRun.state === "failed" && replayedSummary.reconnectRequired ? await deauthorizeAndSettleKnowledgeSourceSyncFailure(db, {
1286
- deauthorization: {
1287
- accountId: input.accountId,
1288
- workspaceId: input.workspaceId,
1289
- sourceId: input.sourceId,
1290
- audience: action.destination,
1291
- operationId: `knowledge-sync-deauthorize:${input.sourceId}:${action.connection.connectionVersion}:${replayedRun.errorCode ?? "connection_reconnect_required"}`,
1292
- reasonCode: "connection_reconnect_required",
1293
- actor: {
1294
- kind: "service",
1295
- subjectId: "knowledge-source-sync",
1296
- initiatingHumanSubjectId: action.initiatingSubjectId
1297
- }
1298
- },
1299
- settlement: settlementInput
1300
- }) : await settleKnowledgeSourceSyncLease(db, settlementInput);
1301
- await recordConnectorUsage("completed", replayedSummary.scanned, "item");
1302
- await recordConnectorUsage("items", replayedSummary.imported, "item");
1303
- await recordConnectorUsage("bytes", replayedSummary.bytes, "byte");
1304
- return {
1305
- action: replayedRun.state === "succeeded" ? "complete" : "failed",
1306
- bufferedWake: settled2.bufferedWake,
1307
- bufferedScheduledTaskRunId: settled2.bufferedScheduledTaskRunId
1308
- };
1309
- }
1310
- const provider = await resolveKnowledgeSyncProvider({
1311
- db,
1312
- settings,
1313
- observability,
1314
- accountId: input.accountId,
1315
- workspaceId: input.workspaceId,
1316
- action,
1317
- sourceId: input.sourceId,
1318
- externalSourceId: resolved.source.externalSourceId
1319
- });
1320
- const selectedDestination = provider.selectedDestination;
1321
- const observationPolicy = knowledgeSyncObservationPolicy(provider.kind);
1322
- let aclGeneration = resolved.source.currentAclGeneration;
1323
- let acl = aclGeneration ? await getKnowledgeSourceAclForSyncAuthority(db, {
1324
- accountId: input.accountId,
1325
- workspaceId: input.workspaceId,
1326
- sourceId: input.sourceId,
1327
- generation: aclGeneration,
1328
- initiatingSubjectId: action.initiatingSubjectId
1329
- }) : null;
1330
- const actor = {
1331
- kind: "service",
1332
- subjectId: "knowledge-source-sync",
1333
- initiatingHumanSubjectId: action.initiatingSubjectId
1334
- };
1335
- if (!acl) {
1336
- acl = await appendKnowledgeSourceAclVersion(db, {
1337
- accountId: input.accountId,
1338
- workspaceId: input.workspaceId,
1339
- sourceId: input.sourceId,
1340
- audience: action.destination,
1341
- expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1342
- expectedAclGeneration: 0,
1343
- aclVersion: "opengeni-destination-v1",
1344
- agentAccess: false,
1345
- operationId: `source-initial-acl:${input.sourceId}`,
1346
- reasonCode: "initial_destination_acl",
1347
- actor
1348
- });
1349
- aclGeneration = acl.generation;
1350
- }
1351
- const knowledgeRun = replayedRun ?? await beginKnowledgeSyncRun(db, {
1352
- accountId: input.accountId,
1353
- workspaceId: input.workspaceId,
1354
- sourceId: input.sourceId,
1355
- expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1356
- expectedSyncGeneration: resolved.source.syncGeneration,
1357
- inputCursor: null,
1358
- operationId: syncOperationId,
1359
- actor
1360
- });
1361
- knowledgeRunId = knowledgeRun.id;
1362
- const driver = provider.driver;
1363
- const inventory = await driver.inventory(
1364
- lease.state.executionCheckpoint,
1365
- lease.state.providerCursor
1366
- );
1367
- summary.providerRequests = inventory.providerRequests;
1368
- summary.elapsedMs = inventory.elapsedMs;
1369
- const base = selectedDestination.collectionId ? await getDocumentBase(db, input.workspaceId, selectedDestination.collectionId) : await ensureDefaultBase(db, {
1370
- accountId: input.accountId,
1371
- workspaceId: input.workspaceId
1372
- });
1373
- if (!base) throw new SyncFailure("authority_changed", false);
1374
- const durableObservations = await recordKnowledgeSourceSyncObjectObservations(db, {
1375
- accountId: input.accountId,
1376
- workspaceId: input.workspaceId,
1377
- sourceId: input.sourceId,
1378
- scheduledTaskRunId: input.scheduledTaskRunId,
1379
- initiatingSubjectId: action.initiatingSubjectId,
1380
- scanGeneration: liveState.activeScanGeneration,
1381
- executionCheckpointGeneration: lease.state.executionCheckpointGeneration,
1382
- revisionOrdering: observationPolicy.revisionOrdering,
1383
- observations: inventory.entries.map((entry) => ({
1384
- externalObjectId: entry.externalObjectId,
1385
- providerRevision: entry.externalVersionId,
1386
- metadataHash: entryMetadataHash(entry)
1387
- }))
1388
- });
1389
- const entriesToProcess = observationPolicy.filterWithDriveDurability ? inventory.entries.filter(
1390
- (entry, index) => shouldProcessGoogleDriveDurableObservation({
1391
- entry,
1392
- observation: durableObservations[index],
1393
- sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1394
- aclGeneration
1395
- })
1396
- ) : inventory.entries;
1397
- const observationFloorsById = /* @__PURE__ */ new Map();
1398
- for (const observation of durableObservations) {
1399
- observationFloorsById.set(observation.externalObjectId, {
1400
- externalObjectId: observation.externalObjectId,
1401
- providerRevision: observation.providerRevision,
1402
- metadataHash: observation.metadataHash
1403
- });
1404
- }
1405
- const observationFloors = [...observationFloorsById.values()];
1406
- const executionCheckpoint = mergeGoogleDriveDurableObservationFloors(
1407
- inventory.checkpoint,
1408
- observationFloors,
1409
- action.limits.maxItems
1410
- );
1411
- const ensureVersionIndexed = async (details) => {
1412
- if (!details.version.documentId) {
1413
- throw new SyncFailure("authority_changed", false);
1414
- }
1415
- let obligation = details.obligation ?? await getKnowledgeSourceSyncIndexObligationForVersion(db, {
1416
- accountId: input.accountId,
1417
- workspaceId: input.workspaceId,
1418
- knowledgeDocumentVersionId: details.version.id,
1419
- initiatingSubjectId: action.initiatingSubjectId
1420
- });
1421
- if (!obligation) {
1422
- obligation = await enqueueKnowledgeSourceSyncIndexObligation(db, {
1423
- accountId: input.accountId,
1424
- workspaceId: input.workspaceId,
1425
- scheduledTaskRunId: input.scheduledTaskRunId,
1426
- sourceId: input.sourceId,
1427
- sourceSyncGeneration: knowledgeRun.inputSyncGeneration,
1428
- initiatingSubjectId: action.initiatingSubjectId,
1429
- externalObjectId: details.entry.externalObjectId,
1430
- knowledgeSourceObjectId: details.object.id,
1431
- knowledgeDocumentVersionId: details.version.id,
1432
- documentId: details.version.documentId,
1433
- sourceConfigGeneration: action.sourceConfigGeneration,
1434
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1435
- objectLifecycleGeneration: details.object.lifecycleGeneration,
1436
- objectVersionGeneration: details.version.versionGeneration,
1437
- citationLocator: driver.citationLocator(details.entry)
1438
- });
1439
- }
1440
- if (obligation.documentId !== details.version.documentId) {
1441
- throw new SyncFailure("authority_changed", false);
1442
- }
1443
- if (obligation.status === "indexed" && !details.indexRequired) return obligation;
1444
- if (obligation.status === "indexed" && details.indexRequired) {
1445
- summary.phase = "index";
1446
- try {
1447
- const indexedDocument = await documentActivities.indexDocument({
1448
- accountId: input.accountId,
1449
- workspaceId: input.workspaceId,
1450
- documentId: details.version.documentId,
1451
- authorityKind: action.destination.kind,
1452
- authorityWorkspaceId: action.destination.workspaceId,
1453
- authoritySubjectId: action.destination.subjectId
1454
- });
1455
- if (indexedDocument.status !== "ready") {
1456
- throw new Error("document indexing did not complete");
1457
- }
1458
- } catch {
1459
- throw new SyncFailure("indexing_failed", true);
1460
- }
1461
- return obligation;
1462
- }
1463
- if (obligation.status === "failed") {
1464
- const retried = await retryKnowledgeSourceSyncIndexObligation(db, {
1465
- accountId: input.accountId,
1466
- workspaceId: input.workspaceId,
1467
- obligationId: obligation.id,
1468
- scheduledTaskRunId: input.scheduledTaskRunId
1469
- });
1470
- if (retried === "invalidated") {
1471
- throw new SyncFailure("authority_changed", false);
1472
- }
1473
- obligation = { ...obligation, status: retried };
1474
- if (retried === "indexed") return obligation;
1475
- }
1476
- if (obligation.status !== "pending") {
1477
- throw new SyncFailure("authority_changed", false);
1478
- }
1479
- if (details.indexRequired) {
1480
- summary.phase = "index";
1481
- try {
1482
- const indexedDocument = await documentActivities.indexDocument({
1483
- accountId: input.accountId,
1484
- workspaceId: input.workspaceId,
1485
- documentId: details.version.documentId,
1486
- authorityKind: action.destination.kind,
1487
- authorityWorkspaceId: action.destination.workspaceId,
1488
- authoritySubjectId: action.destination.subjectId
1489
- });
1490
- if (indexedDocument.status !== "ready") {
1491
- throw new Error("document indexing did not complete");
1492
- }
1493
- } catch {
1494
- await settleKnowledgeSourceSyncIndexObligation(db, {
1495
- accountId: input.accountId,
1496
- workspaceId: input.workspaceId,
1497
- obligationId: obligation.id,
1498
- status: "failed",
1499
- failureCode: "indexing_failed"
1500
- }).catch(() => void 0);
1501
- throw new SyncFailure("indexing_failed", true);
1502
- }
1503
- }
1504
- const settlement = await settleKnowledgeSourceSyncIndexObligation(db, {
1505
- accountId: input.accountId,
1506
- workspaceId: input.workspaceId,
1507
- obligationId: obligation.id,
1508
- status: "indexed"
1509
- });
1510
- if (settlement === "invalidated") {
1511
- throw new SyncFailure("authority_changed", false);
1512
- }
1513
- return { ...obligation, status: "indexed" };
1514
- };
1515
- const recordEntryAcl = async (details) => {
1516
- if (!driver.readAcl || !provider.aclAuthority) {
1517
- return { aclEligibility: "pending", aclEvidence: null };
1518
- }
1519
- await beginGoogleDriveObjectAclRefresh(db, {
1520
- accountId: input.accountId,
1521
- workspaceId: input.workspaceId,
1522
- obligationId: details.obligation.id,
1523
- sourceSyncGeneration: details.obligation.sourceSyncGeneration,
1524
- sourceConfigGeneration: action.sourceConfigGeneration,
1525
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1526
- objectLifecycleGeneration: details.object.lifecycleGeneration,
1527
- objectVersionGeneration: details.version.versionGeneration
1528
- });
1529
- const remainingProviderRequests = action.limits.maxProviderRequests - summary.providerRequests;
1530
- if (remainingProviderRequests < 1) {
1531
- throw new SyncFailure("resource_limit", false);
1532
- }
1533
- const evidence = await driver.readAcl(details.entry, remainingProviderRequests);
1534
- summary.providerRequests += evidence.providerRequests;
1535
- if (summary.providerRequests > action.limits.maxProviderRequests) {
1536
- throw new SyncFailure("resource_limit", false);
1537
- }
1538
- const recorded = await recordGoogleDriveObjectAclEvidence(db, {
1539
- accountId: input.accountId,
1540
- workspaceId: input.workspaceId,
1541
- obligationId: details.obligation.id,
1542
- connectionId: provider.aclAuthority.connectionId,
1543
- connectionVersion: provider.aclAuthority.connectionVersion,
1544
- sourceGooglePermissionId: provider.aclAuthority.sourceGooglePermissionId,
1545
- sourceSyncGeneration: details.obligation.sourceSyncGeneration,
1546
- sourceConfigGeneration: action.sourceConfigGeneration,
1547
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1548
- objectLifecycleGeneration: details.object.lifecycleGeneration,
1549
- objectVersionGeneration: details.version.versionGeneration,
1550
- providerRevision: evidence.providerRevision,
1551
- driveId: evidence.driveId,
1552
- aclRevision: evidence.aclRevision,
1553
- eligibility: evidence.eligibility,
1554
- observedAt: evidence.observedAt,
1555
- expiresAt: evidence.expiresAt,
1556
- citationLocator: driver.citationLocator(details.entry),
1557
- operationId: `drive-acl:${details.obligation.id}:${evidence.aclRevision}`,
1558
- principals: evidence.principals
1559
- });
1560
- return {
1561
- aclEligibility: evidence.eligibility,
1562
- aclEvidence: {
1563
- version: 1,
1564
- provider: "google_drive",
1565
- evidenceId: recorded.evidenceId,
1566
- aclRevision: evidence.aclRevision,
1567
- aclHash: recorded.aclHash,
1568
- observedAt: evidence.observedAt,
1569
- expiresAt: evidence.expiresAt
1570
- }
1571
- };
1572
- };
1573
- const outcomes = [];
1574
- for (const entry of entriesToProcess) {
1575
- heartbeat({ sourceId: input.sourceId, externalObjectId: entry.externalObjectId });
1576
- const observationFloor = observationFloorsById.get(entry.externalObjectId);
1577
- if (!observationFloor) throw new SyncFailure("provider_payload_invalid", false);
1578
- const syncObservation = {
1579
- scheduledTaskRunId: input.scheduledTaskRunId,
1580
- scanGeneration: liveState.activeScanGeneration,
1581
- providerRevision: observationFloor.providerRevision,
1582
- metadataHash: observationFloor.metadataHash
1583
- };
1584
- summary.scanned += 1;
1585
- if (entry.transfer.action === "skip") {
1586
- summary.skipped += 1;
1587
- outcomes.push({
1588
- externalObjectId: entry.externalObjectId,
1589
- outcome: "skipped",
1590
- reasonCode: entry.transfer.reason
1591
- });
1592
- continue;
1593
- }
1594
- if (entry.transfer.action === "traverse") continue;
1595
- try {
1596
- const existingObject = await getKnowledgeSourceObjectForSyncAuthority(db, {
1597
- accountId: input.accountId,
1598
- workspaceId: input.workspaceId,
1599
- sourceId: input.sourceId,
1600
- externalObjectId: entry.externalObjectId,
1601
- initiatingSubjectId: action.initiatingSubjectId
1602
- });
1603
- let object = existingObject ?? await upsertKnowledgeSourceObject(db, {
1604
- accountId: input.accountId,
1605
- workspaceId: input.workspaceId,
1606
- sourceId: input.sourceId,
1607
- externalObjectId: entry.externalObjectId,
1608
- operationId: `source-object:${input.sourceId}:${entry.externalObjectId}`,
1609
- actor
1610
- });
1611
- if (object.lifecycleState !== "active") {
1612
- const restored = await restoreKnowledgeSourceObject(db, {
1613
- accountId: input.accountId,
1614
- workspaceId: input.workspaceId,
1615
- targetId: object.id,
1616
- expectedGeneration: object.lifecycleGeneration,
1617
- operationId: `knowledge-sync-object-restore:${object.id}:${liveState.activeScanGeneration}`,
1618
- reasonCode: "authoritative_scan_observed",
1619
- actor
1620
- });
1621
- object = {
1622
- ...object,
1623
- lifecycleState: "active",
1624
- lifecycleGeneration: restored.lifecycleGeneration
1625
- };
1626
- }
1627
- const currentObservation = object.currentVersionId ? await getKnowledgeDocumentVersionObservationForSyncAuthority(db, {
1628
- accountId: input.accountId,
1629
- workspaceId: input.workspaceId,
1630
- versionId: object.currentVersionId,
1631
- initiatingSubjectId: action.initiatingSubjectId
1632
- }) : null;
1633
- const currentVersion = currentObservation?.version ?? null;
1634
- const metadataHash = entryMetadataHash(entry);
1635
- const observedProviderRevision = typeof currentObservation?.sourceMetadata.providerRevision === "string" ? currentObservation.sourceMetadata.providerRevision : null;
1636
- const observedMetadataHash = typeof currentObservation?.sourceMetadata.metadataHash === "string" ? currentObservation.sourceMetadata.metadataHash : null;
1637
- const observedSourceLifecycleGeneration = Number(
1638
- currentObservation?.sourceMetadata.sourceLifecycleGeneration ?? 0
1639
- );
1640
- const observedObjectLifecycleGeneration = Number(
1641
- currentObservation?.sourceMetadata.objectLifecycleGeneration ?? 0
1642
- );
1643
- const currentObligation = currentVersion ? await getKnowledgeSourceSyncIndexObligationForVersion(db, {
1644
- accountId: input.accountId,
1645
- workspaceId: input.workspaceId,
1646
- knowledgeDocumentVersionId: currentVersion.id,
1647
- initiatingSubjectId: action.initiatingSubjectId
1648
- }) : null;
1649
- if (currentVersion && currentVersion.documentId && observedProviderRevision === entry.externalVersionId && observedMetadataHash === metadataHash && currentVersion.aclGeneration === aclGeneration && observedSourceLifecycleGeneration === resolved.source.lifecycleGeneration && observedObjectLifecycleGeneration === object.lifecycleGeneration && currentObligation?.status !== "invalidated") {
1650
- await updateKnowledgeSourceDocumentObservationMetadata(db, {
1651
- accountId: input.accountId,
1652
- workspaceId: input.workspaceId,
1653
- initiatingSubjectId: action.initiatingSubjectId,
1654
- sourceId: input.sourceId,
1655
- expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1656
- objectId: object.id,
1657
- expectedObjectLifecycleGeneration: object.lifecycleGeneration,
1658
- versionId: currentVersion.id,
1659
- documentId: currentVersion.documentId,
1660
- title: entry.title,
1661
- sourceUri: entry.sourceUri,
1662
- sourceVersion: entry.externalVersionId ?? currentVersion.contentSha256,
1663
- sourceUpdatedAt: entry.modifiedTime,
1664
- syncObservation
1665
- });
1666
- const obligation2 = await ensureVersionIndexed({
1667
- entry,
1668
- object,
1669
- version: currentVersion,
1670
- indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0,
1671
- obligation: currentObligation
1672
- });
1673
- const objectAcl2 = await recordEntryAcl({
1674
- entry,
1675
- object,
1676
- version: currentVersion,
1677
- obligation: obligation2
1678
- });
1679
- summary.unchanged += 1;
1680
- outcomes.push({
1681
- externalObjectId: entry.externalObjectId,
1682
- outcome: "unchanged",
1683
- contentSha256: currentVersion.contentSha256,
1684
- providerRevision: entry.externalVersionId,
1685
- metadataHash,
1686
- aclEligibility: objectAcl2.aclEligibility,
1687
- aclEvidence: objectAcl2.aclEvidence,
1688
- indexObligationId: obligation2.id
1689
- });
1690
- continue;
1691
- }
1692
- summary.phase = "transfer";
1693
- const bytes = await driver.fetchContent(entry, action.limits.maxFileBytes);
1694
- const contentSha256 = createHash2("sha256").update(bytes).digest("hex");
1695
- const observationKey = createHash2("sha256").update(
1696
- JSON.stringify({
1697
- externalObjectId: entry.externalObjectId,
1698
- providerRevision: entry.externalVersionId,
1699
- metadataHash,
1700
- contentSha256,
1701
- aclGeneration,
1702
- sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1703
- objectLifecycleGeneration: object.lifecycleGeneration
1704
- })
1705
- ).digest("hex");
1706
- if (currentVersion?.contentSha256 === contentSha256 && currentVersion.documentId && currentVersion.fileId) {
1707
- const version2 = await appendKnowledgeDocumentVersion(db, {
1708
- accountId: input.accountId,
1709
- workspaceId: input.workspaceId,
1710
- objectId: object.id,
1711
- expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1712
- expectedObjectLifecycleGeneration: object.lifecycleGeneration,
1713
- expectedVersionGeneration: object.versionGeneration,
1714
- externalVersionId: entry.externalVersionId ?? contentSha256,
1715
- contentSha256,
1716
- ingestionKey: `${entry.externalObjectId}:${observationKey}`,
1717
- sourceCursor: null,
1718
- sourceMetadata: {
1719
- title: entry.title,
1720
- mimeType: entry.mimeType,
1721
- driveId: entry.driveId,
1722
- providerRevision: entry.externalVersionId,
1723
- metadataHash,
1724
- aclEligibility: "pending",
1725
- sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1726
- objectLifecycleGeneration: object.lifecycleGeneration
1727
- },
1728
- sourceCreatedAt: entry.createdTime,
1729
- sourceUpdatedAt: entry.modifiedTime,
1730
- aclVersionId: acl.id,
1731
- aclGeneration,
1732
- documentId: currentVersion.documentId,
1733
- fileId: currentVersion.fileId,
1734
- locationMetadata: { sourceUri: entry.sourceUri },
1735
- documentObservationMetadata: {
1736
- title: entry.title,
1737
- sourceUri: entry.sourceUri,
1738
- sourceVersion: entry.externalVersionId ?? contentSha256,
1739
- sourceUpdatedAt: entry.modifiedTime
1740
- },
1741
- syncObservation,
1742
- operationId: `document-observation:${object.id}:${observationKey}`,
1743
- reasonCode: "source_metadata_observed",
1744
- actor
1745
- });
1746
- await updateKnowledgeSourceDocumentObservationMetadata(db, {
1747
- accountId: input.accountId,
1748
- workspaceId: input.workspaceId,
1749
- initiatingSubjectId: action.initiatingSubjectId,
1750
- sourceId: input.sourceId,
1751
- expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1752
- objectId: object.id,
1753
- expectedObjectLifecycleGeneration: object.lifecycleGeneration,
1754
- versionId: version2.id,
1755
- documentId: currentVersion.documentId,
1756
- title: entry.title,
1757
- sourceUri: entry.sourceUri,
1758
- sourceVersion: entry.externalVersionId ?? contentSha256,
1759
- sourceUpdatedAt: entry.modifiedTime,
1760
- syncObservation
1761
- });
1762
- const obligation2 = await ensureVersionIndexed({
1763
- entry,
1764
- object,
1765
- version: version2,
1766
- indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0
1767
- });
1768
- const objectAcl2 = await recordEntryAcl({ entry, object, version: version2, obligation: obligation2 });
1769
- summary.unchanged += 1;
1770
- summary.indexed += 1;
1771
- if (objectAcl2.aclEligibility === "pending") summary.aclPending += 1;
1772
- outcomes.push({
1773
- externalObjectId: entry.externalObjectId,
1774
- outcome: "unchanged",
1775
- contentSha256,
1776
- sizeBytes: bytes.byteLength,
1777
- providerRevision: entry.externalVersionId,
1778
- metadataHash,
1779
- aclEligibility: objectAcl2.aclEligibility,
1780
- aclEvidence: objectAcl2.aclEvidence,
1781
- indexObligationId: obligation2.id
1782
- });
1783
- continue;
1784
- }
1785
- const fileId = deterministicUuid(
1786
- `knowledge-blob:${input.accountId}:${input.workspaceId}:${contentSha256}`
1787
- );
1788
- const objectKey = `workspaces/${input.workspaceId}/knowledge/blobs/${contentSha256}`;
1789
- await objectStorage.putObject({
1790
- key: objectKey,
1791
- contentType: entry.transfer.contentType,
1792
- body: bytes,
1793
- sha256: contentSha256
1794
- });
1795
- const file = await ensureKnowledgeSourceBlobFile(db, {
1796
- accountId: input.accountId,
1797
- workspaceId: input.workspaceId,
1798
- fileId,
1799
- filename: entry.transfer.filename,
1800
- safeFilename: safeFilename(entry.transfer.filename),
1801
- contentType: entry.transfer.contentType,
1802
- sizeBytes: bytes.byteLength,
1803
- sha256: contentSha256,
1804
- bucket: objectStorage.bucket,
1805
- objectKey
1806
- });
1807
- const document = await addDocumentToBase(db, {
1808
- accountId: input.accountId,
1809
- workspaceId: input.workspaceId,
1810
- baseId: base.id,
1811
- fileId: file.id,
1812
- title: entry.title,
1813
- sourceKind: "document",
1814
- sourceUri: entry.sourceUri,
1815
- sourceExternalId: entry.externalObjectId,
1816
- sourceTitle: entry.title,
1817
- sourceCreatedAt: entry.createdTime ?? void 0,
1818
- sourceUpdatedAt: entry.modifiedTime ?? void 0,
1819
- sourceVersion: entry.externalVersionId ?? contentSha256,
1820
- knowledgeSourceIdentity: deterministicUuid(
1821
- `knowledge-source-version:${object.id}:${observationKey}`
1822
- ),
1823
- authorityKind: action.destination.kind,
1824
- createdBy: action.initiatingSubjectId,
1825
- initiatingSubjectId: action.initiatingSubjectId,
1826
- organizationAuthorityGranted: action.destination.kind === "organization",
1827
- agentAccess: false,
1828
- curationStatus: "none",
1829
- access: { viewerSubjectId: action.initiatingSubjectId }
1830
- });
1831
- const version = await appendKnowledgeDocumentVersion(db, {
1832
- accountId: input.accountId,
1833
- workspaceId: input.workspaceId,
1834
- objectId: object.id,
1835
- expectedSourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1836
- expectedObjectLifecycleGeneration: object.lifecycleGeneration,
1837
- expectedVersionGeneration: object.versionGeneration,
1838
- externalVersionId: entry.externalVersionId ?? contentSha256,
1839
- contentSha256,
1840
- ingestionKey: `${entry.externalObjectId}:${observationKey}`,
1841
- sourceCursor: null,
1842
- sourceMetadata: {
1843
- title: entry.title,
1844
- mimeType: entry.mimeType,
1845
- driveId: entry.driveId,
1846
- providerRevision: entry.externalVersionId,
1847
- metadataHash,
1848
- aclEligibility: "pending",
1849
- sourceLifecycleGeneration: resolved.source.lifecycleGeneration,
1850
- objectLifecycleGeneration: object.lifecycleGeneration
1851
- },
1852
- sourceCreatedAt: entry.createdTime,
1853
- sourceUpdatedAt: entry.modifiedTime,
1854
- aclVersionId: acl.id,
1855
- aclGeneration,
1856
- documentId: document.id,
1857
- fileId: file.id,
1858
- locationMetadata: { sourceUri: entry.sourceUri },
1859
- syncObservation,
1860
- operationId: `document-version:${object.id}:${observationKey}`,
1861
- reasonCode: "source_content_observed",
1862
- actor
1863
- });
1864
- const obligation = await ensureVersionIndexed({
1865
- entry,
1866
- object,
1867
- version,
1868
- indexRequired: true
1869
- });
1870
- const objectAcl = await recordEntryAcl({ entry, object, version, obligation });
1871
- summary.indexed += 1;
1872
- summary.imported += 1;
1873
- if (objectAcl.aclEligibility === "pending") summary.aclPending += 1;
1874
- summary.bytes += bytes.byteLength;
1875
- outcomes.push({
1876
- externalObjectId: entry.externalObjectId,
1877
- outcome: "imported",
1878
- contentSha256,
1879
- sizeBytes: bytes.byteLength,
1880
- providerRevision: entry.externalVersionId,
1881
- metadataHash,
1882
- aclEligibility: objectAcl.aclEligibility,
1883
- aclEvidence: objectAcl.aclEvidence,
1884
- indexObligationId: obligation.id
1885
- });
1886
- } catch (error) {
1887
- if (error instanceof KnowledgeSourceSyncObservationFenceError) {
1888
- throw new SyncFailure("provider_payload_invalid", false);
1889
- }
1890
- const message = boundedError(error);
1891
- const itemFailure = error instanceof SyncFailure ? error : new SyncFailure("item_processing_failed", false);
1892
- summary.failed += 1;
1893
- summary.retryable ||= itemFailure.retryable;
1894
- if (summary.failures.length < action.limits.maxFailureDetails) {
1895
- summary.failures.push({
1896
- externalObjectId: entry.externalObjectId,
1897
- code: itemFailure.code,
1898
- retryable: itemFailure.retryable,
1899
- message
1900
- });
1901
- }
1902
- outcomes.push({
1903
- externalObjectId: entry.externalObjectId,
1904
- outcome: "failed",
1905
- reasonCode: itemFailure.code,
1906
- detail: message,
1907
- providerRevision: entry.externalVersionId,
1908
- metadataHash: entryMetadataHash(entry),
1909
- aclEligibility: "pending"
1910
- });
1911
- }
1912
- }
1913
- await recordKnowledgeSourceSyncItemOutcomes(db, {
1914
- accountId: input.accountId,
1915
- workspaceId: input.workspaceId,
1916
- scheduledTaskRunId: input.scheduledTaskRunId,
1917
- knowledgeSyncRunId: knowledgeRun.id,
1918
- sourceId: input.sourceId,
1919
- sourceConfigGeneration: action.sourceConfigGeneration,
1920
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1921
- outcomes
1922
- });
1923
- if (inventory.status === "paused" && inventory.checkpoint) {
1924
- if (inventory.stopReason === "provider_error" || inventory.stopReason === "incomplete_search") {
1925
- throw new SyncFailure(
1926
- inventory.stopReason === "provider_error" ? "provider_unavailable" : "provider_payload_invalid",
1927
- inventory.stopReason === "provider_error"
1928
- );
1929
- }
1930
- if (inventory.hardLimitReached) {
1931
- summary.limitReached = inventoryLimit(inventory.stopReason);
1932
- throw new SyncFailure("resource_limit", false);
1933
- }
1934
- if (inventory.stopReason !== "elapsed_time_limit") {
1935
- summary.limitReached = inventoryLimit(inventory.stopReason);
1936
- throw new SyncFailure("resource_limit", false);
1937
- }
1938
- summary.checkpointed = true;
1939
- summary.phase = "checkpoint";
1940
- await checkpointKnowledgeSourceSync(db, {
1941
- accountId: input.accountId,
1942
- workspaceId: input.workspaceId,
1943
- sourceId: input.sourceId,
1944
- scheduledTaskRunId: input.scheduledTaskRunId,
1945
- sourceConfigGeneration: action.sourceConfigGeneration,
1946
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1947
- executionCheckpoint,
1948
- observationFence: {
1949
- initiatingSubjectId: action.initiatingSubjectId,
1950
- scanGeneration: liveState.activeScanGeneration,
1951
- executionCheckpointGeneration: lease.state.executionCheckpointGeneration,
1952
- observations: observationFloors
1953
- }
1954
- });
1955
- return { action: "continue" };
1956
- }
1957
- const tombstonedExternalObjectIds = inventory.authoritativeFullScan ? await reconcileKnowledgeSourceSyncCompleteScan(db, {
1958
- accountId: input.accountId,
1959
- workspaceId: input.workspaceId,
1960
- sourceId: input.sourceId,
1961
- scheduledTaskRunId: input.scheduledTaskRunId,
1962
- initiatingSubjectId: action.initiatingSubjectId,
1963
- sourceSyncGeneration: knowledgeRun.inputSyncGeneration,
1964
- sourceConfigGeneration: action.sourceConfigGeneration,
1965
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1966
- scanGeneration: liveState.activeScanGeneration
1967
- }) : [];
1968
- if (tombstonedExternalObjectIds.length > 0) {
1969
- summary.skipped += tombstonedExternalObjectIds.length;
1970
- await recordKnowledgeSourceSyncItemOutcomes(db, {
1971
- accountId: input.accountId,
1972
- workspaceId: input.workspaceId,
1973
- scheduledTaskRunId: input.scheduledTaskRunId,
1974
- knowledgeSyncRunId: knowledgeRun.id,
1975
- sourceId: input.sourceId,
1976
- sourceConfigGeneration: action.sourceConfigGeneration,
1977
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
1978
- outcomes: tombstonedExternalObjectIds.map((externalObjectId) => ({
1979
- externalObjectId,
1980
- outcome: "tombstoned",
1981
- reasonCode: "authoritative_scan_absent",
1982
- aclEligibility: "denied"
1983
- }))
1984
- });
1985
- }
1986
- await completeKnowledgeSyncRun(db, {
1987
- accountId: input.accountId,
1988
- workspaceId: input.workspaceId,
1989
- initiatingSubjectId: action.initiatingSubjectId,
1990
- runId: knowledgeRun.id,
1991
- state: "succeeded",
1992
- outputCursor: null,
1993
- watermark: (/* @__PURE__ */ new Date()).toISOString(),
1994
- metadata: { ...summary, providerCursor: inventory.providerCursor },
1995
- reasonCode: "scheduled_source_sync"
1996
- });
1997
- summary.phase = "completed";
1998
- summary.elapsedMs = Date.now() - startedAt;
1999
- const settled = await settleKnowledgeSourceSyncLease(db, {
2000
- accountId: input.accountId,
2001
- workspaceId: input.workspaceId,
2002
- sourceId: input.sourceId,
2003
- scheduledTaskRunId: input.scheduledTaskRunId,
2004
- knowledgeSyncRunId: knowledgeRun.id,
2005
- status: "succeeded",
2006
- summary,
2007
- sourceConfigGeneration: action.sourceConfigGeneration,
2008
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
2009
- sourceSyncGeneration: knowledgeRun.inputSyncGeneration,
2010
- completedSourceSyncGeneration: knowledgeRun.inputSyncGeneration + 1,
2011
- providerCursor: inventory.providerCursor,
2012
- observationFence: {
2013
- initiatingSubjectId: action.initiatingSubjectId,
2014
- scanGeneration: liveState.activeScanGeneration,
2015
- executionCheckpointGeneration: lease.state.executionCheckpointGeneration,
2016
- observations: observationFloors
2017
- }
2018
- });
2019
- await recordConnectorUsage("completed", summary.scanned, "item");
2020
- await recordConnectorUsage("items", summary.imported, "item");
2021
- await recordConnectorUsage("bytes", summary.bytes, "byte");
2022
- observability.incrementCounter({
2023
- name: "opengeni_knowledge_source_sync_runs_total",
2024
- labels: { provider: providerLabel, outcome: "succeeded" }
2025
- });
2026
- observability.incrementCounter({
2027
- name: "opengeni_knowledge_source_sync_items_total",
2028
- labels: { provider: providerLabel, outcome: "imported" },
2029
- amount: summary.imported
2030
- });
2031
- for (const outcome of ["unchanged", "skipped", "failed"]) {
2032
- observability.incrementCounter({
2033
- name: "opengeni_knowledge_source_sync_items_total",
2034
- labels: { provider: providerLabel, outcome },
2035
- amount: summary[outcome]
2036
- });
2037
- }
2038
- observability.incrementCounter({
2039
- name: "opengeni_knowledge_source_sync_bytes_total",
2040
- labels: { provider: providerLabel },
2041
- amount: summary.bytes
2042
- });
2043
- recordSyncHealthTelemetry(observability, providerLabel, summary, "succeeded");
2044
- return {
2045
- action: "complete",
2046
- bufferedWake: settled.bufferedWake,
2047
- bufferedScheduledTaskRunId: settled.bufferedScheduledTaskRunId
2048
- };
2049
- } catch (error) {
2050
- const failure = error instanceof SyncFailure ? error : new SyncFailure("internal_failure", false);
2051
- summary.phase = "failed";
2052
- summary.retryable ||= failure.retryable;
2053
- summary.elapsedMs = Date.now() - startedAt;
2054
- summary.reconnectRequired = failure.reconnectRequired;
2055
- summary.failed = Math.max(summary.failed, 1);
2056
- if (summary.failures.length === 0) {
2057
- summary.failures.push({
2058
- externalObjectId: input.sourceId,
2059
- code: failure.code,
2060
- retryable: failure.retryable,
2061
- message: boundedError(error)
2062
- });
2063
- }
2064
- if (knowledgeRunId) {
2065
- await completeKnowledgeSyncRun(db, {
2066
- accountId: input.accountId,
2067
- workspaceId: input.workspaceId,
2068
- initiatingSubjectId: task.action.initiatingSubjectId,
2069
- runId: knowledgeRunId,
2070
- state: "failed",
2071
- metadata: summary,
2072
- errorCode: failure.code,
2073
- reasonCode: "scheduled_source_sync_failed"
2074
- }).catch(() => void 0);
2075
- }
2076
- const settlementInput = {
2077
- accountId: input.accountId,
2078
- workspaceId: input.workspaceId,
2079
- sourceId: input.sourceId,
2080
- scheduledTaskRunId: input.scheduledTaskRunId,
2081
- knowledgeSyncRunId: knowledgeRunId,
2082
- status: "failed",
2083
- summary,
2084
- error: failure.code,
2085
- sourceConfigGeneration: action.sourceConfigGeneration,
2086
- sourceLifecycleGeneration: action.sourceLifecycleGeneration,
2087
- sourceSyncGeneration: activeSourceSyncGeneration ?? lease.state.sourceSyncGeneration,
2088
- // Failed or incomplete provider walks are not authoritative scans.
2089
- // A later scheduled run must restart from the root under a fresh
2090
- // observation generation before it can infer object absence.
2091
- executionCheckpoint: null
2092
- };
2093
- const settled = failure.reconnectRequired ? await deauthorizeAndSettleKnowledgeSourceSyncFailure(db, {
2094
- deauthorization: {
2095
- accountId: input.accountId,
2096
- workspaceId: input.workspaceId,
2097
- sourceId: input.sourceId,
2098
- audience: action.destination,
2099
- operationId: `knowledge-sync-deauthorize:${input.sourceId}:${action.connection.connectionVersion}:${failure.code}`,
2100
- reasonCode: "connection_reconnect_required",
2101
- actor: {
2102
- kind: "service",
2103
- subjectId: "knowledge-source-sync",
2104
- initiatingHumanSubjectId: action.initiatingSubjectId
2105
- }
2106
- },
2107
- settlement: settlementInput
2108
- }) : await settleKnowledgeSourceSyncLease(db, settlementInput);
2109
- await recordConnectorUsage("completed", summary.scanned, "item");
2110
- await recordConnectorUsage("items", summary.imported, "item");
2111
- await recordConnectorUsage("bytes", summary.bytes, "byte");
2112
- observability.incrementCounter({
2113
- name: "opengeni_knowledge_source_sync_runs_total",
2114
- labels: { provider: providerLabel, outcome: "failed" }
2115
- });
2116
- for (const outcome of ["imported", "unchanged", "skipped", "failed"]) {
2117
- observability.incrementCounter({
2118
- name: "opengeni_knowledge_source_sync_items_total",
2119
- labels: { provider: providerLabel, outcome },
2120
- amount: summary[outcome]
2121
- });
2122
- }
2123
- observability.incrementCounter({
2124
- name: "opengeni_knowledge_source_sync_bytes_total",
2125
- labels: { provider: providerLabel },
2126
- amount: summary.bytes
2127
- });
2128
- recordSyncHealthTelemetry(observability, providerLabel, summary, "failed", failure);
2129
- return {
2130
- action: "failed",
2131
- bufferedWake: settled.bufferedWake,
2132
- bufferedScheduledTaskRunId: settled.bufferedScheduledTaskRunId
2133
- };
2134
- }
2135
- async function failWithoutProvider(code) {
2136
- const terminalSummary = {
2137
- phase: "failed",
2138
- scanned: 0,
2139
- imported: 0,
2140
- unchanged: 0,
2141
- skipped: 0,
2142
- failed: 1,
2143
- bytes: 0,
2144
- providerRequests: 0,
2145
- elapsedMs: 0,
2146
- indexed: 0,
2147
- aclPending: 0,
2148
- retryable: false,
2149
- limitReached: null,
2150
- checkpointed: false,
2151
- reconnectRequired: false,
2152
- failures: [
2153
- {
2154
- externalObjectId: input.sourceId,
2155
- code: "authority_changed",
2156
- retryable: false,
2157
- message: code
2158
- }
2159
- ]
2160
- };
2161
- await updateScheduledTaskRun(db, input.workspaceId, input.scheduledTaskRunId, {
2162
- status: "failed",
2163
- actionKind: "knowledge_source_sync",
2164
- knowledgeSummary: terminalSummary,
2165
- completedAt: /* @__PURE__ */ new Date(),
2166
- error: code
2167
- }).catch(() => void 0);
2168
- await completeKnowledgeSourceSyncWakeWithoutLease(db, {
2169
- accountId: input.accountId,
2170
- workspaceId: input.workspaceId,
2171
- sourceId: input.sourceId,
2172
- scheduledTaskRunId: input.scheduledTaskRunId
2173
- }).catch(() => void 0);
2174
- await recordConnectorUsage("completed", 0, "item").catch(() => void 0);
2175
- observability.incrementCounter({
2176
- name: "opengeni_knowledge_source_sync_runs_total",
2177
- labels: { provider: "unknown", outcome: "failed_preflight" }
2178
- });
2179
- return { action: "failed", bufferedWake: false };
2180
- }
2181
- async function recordConnectorUsage(kind, quantity, unit) {
2182
- if (quantity === 0 && kind !== "completed") return;
2183
- await recordUsageEvent2(db, {
2184
- accountId: input.accountId,
2185
- workspaceId: input.workspaceId,
2186
- eventType: `knowledge_source_sync.${kind}`,
2187
- quantity: kind === "completed" ? 1 : quantity,
2188
- unit,
2189
- sourceResourceType: "scheduled_task_run",
2190
- sourceResourceId: input.scheduledTaskRunId,
2191
- initiator: { kind: "service", subjectId: "knowledge-source-sync" },
2192
- initiatorContext: {
2193
- scheduledTaskId: input.taskId,
2194
- scheduledTaskRunId: input.scheduledTaskRunId
2195
- },
2196
- origin: "scheduled_task",
2197
- idempotencyKey: `usage:knowledge_source_sync.${kind}:${input.scheduledTaskRunId}`
2198
- });
2199
- }
2200
- }
2201
- };
2202
- }
2203
- async function deauthorizeAndSettleKnowledgeSourceSyncFailure(db, input) {
2204
- try {
2205
- await deauthorizeKnowledgeSourceRetrieval(db, input.deauthorization);
2206
- } catch (error) {
2207
- await releaseKnowledgeSourceSyncLeaseForRetry(db, {
2208
- accountId: input.settlement.accountId,
2209
- workspaceId: input.settlement.workspaceId,
2210
- sourceId: input.settlement.sourceId,
2211
- scheduledTaskRunId: input.settlement.scheduledTaskRunId
2212
- });
2213
- throw error;
2214
- }
2215
- return await settleKnowledgeSourceSyncLease(db, input.settlement);
2216
- }
2217
- async function resolveKnowledgeSyncProvider(input) {
2218
- const { action } = input;
2219
- const connection = await getConnectionMetadata(
2220
- input.db,
2221
- input.workspaceId,
2222
- action.connection.connectionId,
2223
- action.initiatingSubjectId
2224
- );
2225
- if (!connection || connection.accountId !== input.accountId || connection.subjectId !== action.connection.ownerSubjectId || connection.providerDomain.toLowerCase() !== action.connection.providerDomain.toLowerCase() || connection.kind !== action.connection.kind || connection.version < action.connection.connectionVersion || connection.status !== "active") {
2226
- throw new SyncFailure("connection_reconnect_required", true, true);
2227
- }
2228
- const googleInitial = connection.providerDomain === GOOGLE_DRIVE_PROVIDER_DOMAIN ? GoogleDriveConnectionMetadata.safeParse(connection.metadata) : null;
2229
- const atlassianInitial = connection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN ? AtlassianConnectionMetadata.safeParse(connection.metadata) : null;
2230
- const initialAtlassianMetadata = atlassianInitial?.success ? atlassianInitial.data : null;
2231
- if (!googleInitial?.success && !initialAtlassianMetadata) {
2232
- throw new SyncFailure("provider_rejected", false);
2233
- }
2234
- if (googleInitial?.success && !googleDriveScopesAllowCapability(connection.grantedScopes, "recursive_source_sync")) {
2235
- throw new SyncFailure("connection_reconnect_required", true, true);
2236
- }
2237
- if (atlassianInitial?.success && !atlassianScopesAllowRead(connection.grantedScopes)) {
2238
- throw new SyncFailure("connection_reconnect_required", true, true);
2239
- }
2240
- const initialLifecycle = googleInitial?.success ? googleInitial.data.lifecycle : initialAtlassianMetadata.lifecycle;
2241
- if (initialLifecycle?.state && initialLifecycle.state !== "active") {
2242
- throw new SyncFailure("connection_reconnect_required", true, true);
2243
- }
2244
- const initialSource = googleInitial?.success ? selectedDriveSource(googleInitial.data, input.externalSourceId) : selectedAtlassianSource(initialAtlassianMetadata, input.externalSourceId);
2245
- if (!initialSource || !initialSource.syncEnabled || initialSource.configGeneration !== action.sourceConfigGeneration || !initialSource.destination || scopedKnowledgeScopeKey(
2246
- googleInitial?.success ? googleDriveKnowledgeScope(initialSource.destination) : atlassianKnowledgeScope(initialSource.destination)
2247
- ) !== scopedKnowledgeScopeKey(action.destination)) {
2248
- throw new SyncFailure("authority_changed", false);
2249
- }
2250
- const token = await buildConnectionTokenResolver(
2251
- input.db,
2252
- input.settings
2253
- )({
2254
- workspaceId: input.workspaceId,
2255
- subjectId: action.initiatingSubjectId,
2256
- serverId: "knowledge-source-sync",
2257
- connectionRef: {
2258
- connectionId: connection.id,
2259
- providerDomain: connection.providerDomain,
2260
- kind: connection.kind,
2261
- subjectScope: "subject"
2262
- },
2263
- destinationUrl: connection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN ? ATLASSIAN_API_BASE : "https://www.googleapis.com"
2264
- });
2265
- if (token.status !== "ok") {
2266
- throw new SyncFailure("connection_reconnect_required", true, true);
2267
- }
2268
- const resolvedConnection = await getConnectionMetadata(
2269
- input.db,
2270
- input.workspaceId,
2271
- action.connection.connectionId,
2272
- action.initiatingSubjectId
2273
- );
2274
- if (!resolvedConnection || resolvedConnection.accountId !== input.accountId || resolvedConnection.subjectId !== action.connection.ownerSubjectId || resolvedConnection.providerDomain.toLowerCase() !== action.connection.providerDomain.toLowerCase() || resolvedConnection.kind !== action.connection.kind || resolvedConnection.version !== token.connectionVersion || resolvedConnection.status !== "active") {
2275
- throw new SyncFailure("connection_reconnect_required", true, true);
2276
- }
2277
- const authorization = token.headers.Authorization ?? token.headers.authorization;
2278
- if (resolvedConnection.providerDomain === GOOGLE_DRIVE_PROVIDER_DOMAIN) {
2279
- const metadata = GoogleDriveConnectionMetadata.parse(resolvedConnection.metadata);
2280
- if (!googleDriveSyncProviderAccessAllowed({
2281
- connectionVersion: resolvedConnection.version,
2282
- resolvedConnectionVersion: token.connectionVersion,
2283
- connectionStatus: resolvedConnection.status,
2284
- lifecycleState: metadata.lifecycle?.state,
2285
- grantedScopes: resolvedConnection.grantedScopes
2286
- })) {
2287
- throw new SyncFailure("connection_reconnect_required", true, true);
2288
- }
2289
- const source = selectedDriveSource(metadata, input.externalSourceId);
2290
- if (!validSelectedSource(source, action, googleDriveKnowledgeScope)) {
2291
- throw new SyncFailure("authority_changed", false);
2292
- }
2293
- return {
2294
- kind: "google_drive",
2295
- selectedDestination: source.destination,
2296
- aclAuthority: {
2297
- connectionId: resolvedConnection.id,
2298
- connectionVersion: resolvedConnection.version,
2299
- sourceGooglePermissionId: metadata.googlePermissionId
2300
- },
2301
- driver: googleDriveSyncDriver({
2302
- actionProviderCoordinationKey: action.providerCoordinationKey,
2303
- metadata,
2304
- selectedSource: source,
2305
- accountId: input.accountId,
2306
- workspaceId: input.workspaceId,
2307
- initiatingSubjectId: action.initiatingSubjectId,
2308
- authorization,
2309
- retry: googleDriveProviderRetryOptions(input.settings),
2310
- observability: input.observability,
2311
- limits: action.limits,
2312
- connectionId: resolvedConnection.id,
2313
- fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
2314
- observedExternalObjectIds: async (externalObjectIds) => await listObservedKnowledgeSourceSyncExternalObjectIds(input.db, {
2315
- accountId: input.accountId,
2316
- workspaceId: input.workspaceId,
2317
- sourceId: input.sourceId,
2318
- externalObjectIds
2319
- })
2320
- })
2321
- };
2322
- }
2323
- if (resolvedConnection.providerDomain === ATLASSIAN_PROVIDER_DOMAIN) {
2324
- const metadata = AtlassianConnectionMetadata.parse(resolvedConnection.metadata);
2325
- if (metadata.lifecycle?.state !== "active" || !atlassianScopesAllowRead(resolvedConnection.grantedScopes)) {
2326
- throw new SyncFailure("connection_reconnect_required", true, true);
2327
- }
2328
- const source = selectedAtlassianSource(metadata, input.externalSourceId);
2329
- if (!validSelectedSource(source, action, atlassianKnowledgeScope)) {
2330
- throw new SyncFailure("authority_changed", false);
2331
- }
2332
- return {
2333
- kind: "atlassian",
2334
- selectedDestination: source.destination,
2335
- aclAuthority: null,
2336
- driver: atlassianSyncDriver({
2337
- actionProviderCoordinationKey: action.providerCoordinationKey,
2338
- metadata,
2339
- selectedSource: source,
2340
- authorization,
2341
- limits: action.limits
2342
- })
2343
- };
2344
- }
2345
- throw new SyncFailure("provider_rejected", false);
2346
- }
2347
- function validSelectedSource(source, action, scope) {
2348
- return Boolean(
2349
- source && source.syncEnabled && source.configGeneration === action.sourceConfigGeneration && source.destination && scopedKnowledgeScopeKey(scope(source.destination)) === scopedKnowledgeScopeKey(action.destination)
2350
- );
2351
- }
2352
- function selectedAtlassianSource(metadata, externalSourceId) {
2353
- return metadata.selectedSources.find((source) => source.id === externalSourceId) ?? null;
2354
- }
2355
- function atlassianSyncDriver(input) {
2356
- return {
2357
- providerKey: "atlassian",
2358
- providerDomain: ATLASSIAN_PROVIDER_DOMAIN,
2359
- providerCoordinationKey: input.actionProviderCoordinationKey,
2360
- inventory: async (checkpoint) => {
2361
- const inventory = await inventoryAtlassianSource({
2362
- cloudId: input.selectedSource.cloudId,
2363
- source: input.selectedSource,
2364
- limits: {
2365
- maxItems: input.limits.maxItems,
2366
- maxApiRequests: input.limits.maxProviderRequests,
2367
- maxElapsedMs: input.limits.maxElapsedSeconds * 1e3,
2368
- pageSize: Math.min(input.limits.maxItems, 100)
2369
- },
2370
- checkpoint,
2371
- listPage: async (cursor, pageSize) => await listAtlassianItems(input.selectedSource, cursor, pageSize, input.authorization)
2372
- });
2373
- return {
2374
- ...inventory,
2375
- providerCursor: null,
2376
- authoritativeFullScan: inventory.status === "complete",
2377
- cursorInvalidated: false,
2378
- hardLimitReached: inventory.status === "paused" && inventory.stopReason !== "elapsed_time_limit" && inventory.stopReason !== "provider_error"
2379
- };
2380
- },
2381
- fetchContent: async (entry, maxBytes) => await fetchAtlassianContent(input.selectedSource, entry, input.authorization, maxBytes),
2382
- citationLocator: (entry) => ({
2383
- version: 1,
2384
- providerKey: "atlassian",
2385
- providerCoordinationKey: input.actionProviderCoordinationKey,
2386
- cloudId: input.selectedSource.cloudId,
2387
- sourceKind: input.selectedSource.kind,
2388
- externalObjectId: entry.externalObjectId,
2389
- sourceUri: entry.sourceUri
2390
- })
2391
- };
2392
- }
2393
- function googleDriveSyncDriver(input) {
2394
- const maxElapsedMs = input.limits.maxElapsedSeconds * 1e3;
2395
- const provider = input.provider ?? {
2396
- getStartPageToken: async (driveId) => await getDriveStartPageToken(driveId, input.authorization, input.retry, input.observability),
2397
- listChanges: async (pageToken, driveId, pageSize) => await listDriveChanges(
2398
- pageToken,
2399
- driveId,
2400
- pageSize,
2401
- input.authorization,
2402
- input.retry,
2403
- input.observability
2404
- ),
2405
- getFile: async (fileId) => await getDriveFile(fileId, input.authorization, input.retry, input.observability),
2406
- listChildren: async (request) => await listDriveChildren(request, input.authorization, input.retry, input.observability),
2407
- listPermissions: async (request) => await listDrivePermissions(request, input.authorization, input.retry, input.observability)
2408
- };
2409
- const clock = provider.now ?? Date.now;
2410
- const budgetLimits = {
2411
- maxItems: input.limits.maxItems,
2412
- maxProviderRequests: input.limits.maxProviderRequests,
2413
- maxElapsedMs
2414
- };
2415
- return {
2416
- providerKey: "google_drive",
2417
- providerDomain: GOOGLE_DRIVE_PROVIDER_DOMAIN,
2418
- providerCoordinationKey: input.actionProviderCoordinationKey,
2419
- inventory: async (executionCheckpoint, providerCursor) => {
2420
- const expectedCursor = {
2421
- connectionId: input.connectionId,
2422
- googlePermissionId: input.metadata.googlePermissionId,
2423
- sourceId: input.selectedSource.id,
2424
- driveId: input.selectedSource.driveId
2425
- };
2426
- const legacyFullCheckpoint = isLegacyGoogleDriveFullReconciliationCheckpoint(
2427
- executionCheckpoint,
2428
- expectedCursor
2429
- );
2430
- const cursor = parseGoogleDriveChangesCursor(providerCursor, expectedCursor);
2431
- const checkpoint = parseGoogleDriveExecutionCheckpoint(
2432
- executionCheckpoint,
2433
- expectedCursor,
2434
- budgetLimits
2435
- );
2436
- const now = new Date(clock());
2437
- const runFullReconciliation = async (full) => {
2438
- const inventoryItems = full.inventoryCheckpoint?.totals.itemCount ?? 0;
2439
- const inventoryProviderRequests = full.inventoryCheckpoint?.totals.apiRequestCount ?? 0;
2440
- const budgetBeforeInvocation = {
2441
- examinedItems: full.budgetBeforeInventory.examinedItems + inventoryItems,
2442
- providerRequests: full.budgetBeforeInventory.providerRequests + inventoryProviderRequests,
2443
- elapsedMs: full.budgetBeforeInventory.elapsedMs + full.inventoryElapsedMs
2444
- };
2445
- const hardStopReason = googleDriveHardLimitReason(budgetBeforeInvocation, budgetLimits);
2446
- const fullCheckpoint = buildGoogleDriveFullReconciliationCheckpoint({
2447
- ...expectedCursor,
2448
- ...full
2449
- });
2450
- if (hardStopReason) {
2451
- return {
2452
- status: "paused",
2453
- stopReason: hardStopReason,
2454
- entries: [],
2455
- checkpoint: fullCheckpoint,
2456
- providerCursor,
2457
- authoritativeFullScan: false,
2458
- cursorInvalidated: full.cursorInvalidated,
2459
- providerRequests: budgetBeforeInvocation.providerRequests,
2460
- elapsedMs: budgetBeforeInvocation.elapsedMs,
2461
- hardLimitReached: true
2462
- };
2463
- }
2464
- const phaseMaxItems = input.limits.maxItems - full.budgetBeforeInventory.examinedItems;
2465
- const phaseMaxProviderRequests = input.limits.maxProviderRequests - full.budgetBeforeInventory.providerRequests;
2466
- const remainingElapsedMs = maxElapsedMs - budgetBeforeInvocation.elapsedMs;
2467
- const inventory = await inventoryGoogleDriveSource({
2468
- googlePermissionId: input.metadata.googlePermissionId,
2469
- googleEmail: input.metadata.googleEmail,
2470
- source: input.selectedSource,
2471
- accountId: input.accountId,
2472
- workspaceId: input.workspaceId,
2473
- connectionSubjectId: input.initiatingSubjectId,
2474
- limits: {
2475
- maxItems: phaseMaxItems,
2476
- maxKnownBytes: input.limits.maxBytes,
2477
- maxApiRequests: phaseMaxProviderRequests,
2478
- maxElapsedMs: Math.min(GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS, remainingElapsedMs),
2479
- maxFileBytes: input.limits.maxFileBytes,
2480
- maxFolders: Math.min(phaseMaxItems * 2, 2e3),
2481
- pageSize: Math.min(phaseMaxItems, 100)
2482
- },
2483
- checkpoint: full.inventoryCheckpoint,
2484
- listChildren: provider.listChildren,
2485
- now: clock
2486
- });
2487
- const inventoryElapsedMs = full.inventoryElapsedMs + inventory.run.elapsedMs;
2488
- const budget = {
2489
- examinedItems: full.budgetBeforeInventory.examinedItems + inventory.totals.itemCount,
2490
- providerRequests: full.budgetBeforeInventory.providerRequests + inventory.totals.apiRequestCount,
2491
- elapsedMs: full.budgetBeforeInventory.elapsedMs + inventoryElapsedMs
2492
- };
2493
- const reconciled = reconcileGoogleDriveRevisionFloors(
2494
- inventory.entries,
2495
- full.revisionFloors,
2496
- input.limits.maxItems
2497
- );
2498
- if (reconciled.conflict) {
2499
- return {
2500
- status: "paused",
2501
- stopReason: "incomplete_search",
2502
- entries: [],
2503
- checkpoint: fullCheckpoint,
2504
- providerCursor,
2505
- authoritativeFullScan: false,
2506
- cursorInvalidated: full.cursorInvalidated,
2507
- providerRequests: budget.providerRequests,
2508
- elapsedMs: budget.elapsedMs,
2509
- hardLimitReached: false
2510
- };
2511
- }
2512
- const nextCursor = inventory.status === "complete" ? buildGoogleDriveChangesCursor({
2513
- ...expectedCursor,
2514
- boundaryId: full.boundaryId,
2515
- pageToken: full.startPageToken,
2516
- previousGeneration: cursor?.cursorGeneration ?? 0,
2517
- reconciledAt: now,
2518
- fullReconciliationIntervalMs: input.fullReconciliationIntervalMs
2519
- }) : providerCursor;
2520
- return {
2521
- status: inventory.status,
2522
- stopReason: inventory.stopReason,
2523
- entries: reconciled.entries,
2524
- checkpoint: inventory.status === "paused" && inventory.checkpoint ? buildGoogleDriveFullReconciliationCheckpoint({
2525
- ...expectedCursor,
2526
- boundaryId: full.boundaryId,
2527
- startPageToken: full.startPageToken,
2528
- cursorInvalidated: full.cursorInvalidated,
2529
- budgetBeforeInventory: full.budgetBeforeInventory,
2530
- inventoryElapsedMs,
2531
- inventoryCheckpoint: inventory.checkpoint,
2532
- revisionFloors: reconciled.revisionFloors
2533
- }) : null,
2534
- providerCursor: nextCursor,
2535
- authoritativeFullScan: inventory.status === "complete",
2536
- cursorInvalidated: full.cursorInvalidated,
2537
- providerRequests: budget.providerRequests,
2538
- elapsedMs: budget.elapsedMs,
2539
- hardLimitReached: inventory.status === "paused" && googleDriveInventoryStopIsHard(inventory.stopReason, budget, budgetLimits)
2540
- };
2541
- };
2542
- const acquireFullReconciliation = async (details) => {
2543
- const acquisitionStartedAt = clock();
2544
- const initialElapsedMs = details.budget.elapsedMs;
2545
- const budget = { ...details.budget };
2546
- const refreshElapsed = () => {
2547
- budget.elapsedMs = initialElapsedMs + Math.max(0, clock() - acquisitionStartedAt);
2548
- };
2549
- const requireProviderCapacity = () => {
2550
- refreshElapsed();
2551
- if (googleDriveHardLimitReason(budget, budgetLimits)) {
2552
- throw new SyncFailure("resource_limit", false);
2553
- }
2554
- budget.providerRequests += 1;
2555
- };
2556
- requireProviderCapacity();
2557
- const startPageToken = await provider.getStartPageToken(input.selectedSource.driveId);
2558
- refreshElapsed();
2559
- let boundaryId = details.boundaryId;
2560
- if (!boundaryId) {
2561
- if (input.selectedSource.id === "root" && input.selectedSource.driveId === null) {
2562
- requireProviderCapacity();
2563
- try {
2564
- boundaryId = await resolveGoogleDriveChangesBoundaryId({
2565
- source: input.selectedSource,
2566
- getFile: provider.getFile
2567
- });
2568
- } catch (error) {
2569
- if (error instanceof GoogleDriveChangesProtocolError) {
2570
- throw new SyncFailure("provider_payload_invalid", false);
2571
- }
2572
- throw error;
2573
- }
2574
- refreshElapsed();
2575
- } else {
2576
- boundaryId = input.selectedSource.id;
2577
- }
2578
- }
2579
- return {
2580
- boundaryId,
2581
- startPageToken,
2582
- cursorInvalidated: details.cursorInvalidated,
2583
- budgetBeforeInventory: budget,
2584
- inventoryElapsedMs: 0,
2585
- inventoryCheckpoint: null,
2586
- revisionFloors: []
2587
- };
2588
- };
2589
- if (checkpoint?.kind === "google_drive_full_reconciliation") {
2590
- return await runFullReconciliation(checkpoint);
2591
- }
2592
- if (!cursor) {
2593
- const budget = checkpoint?.kind === "google_drive_changes" ? checkpoint.changesCheckpoint.budget : emptyGoogleDriveSyncBudget();
2594
- return await runFullReconciliation(
2595
- await acquireFullReconciliation({
2596
- budget,
2597
- boundaryId: null,
2598
- cursorInvalidated: providerCursor !== null
2599
- })
2600
- );
2601
- }
2602
- if (!checkpoint && !legacyFullCheckpoint && googleDriveFullReconciliationDue(cursor, now)) {
2603
- return await runFullReconciliation(
2604
- await acquireFullReconciliation({
2605
- budget: emptyGoogleDriveSyncBudget(),
2606
- boundaryId: cursor.boundaryId,
2607
- cursorInvalidated: false
2608
- })
2609
- );
2610
- }
2611
- try {
2612
- const forceFullReconciliation = legacyFullCheckpoint || googleDriveFullReconciliationDue(cursor, now);
2613
- const changesCheckpoint = checkpoint?.kind === "google_drive_changes" ? {
2614
- ...checkpoint.changesCheckpoint,
2615
- requiresFullReconciliation: checkpoint.changesCheckpoint.requiresFullReconciliation || forceFullReconciliation
2616
- } : null;
2617
- const changes = await drainGoogleDriveChanges({
2618
- source: input.selectedSource,
2619
- cursor,
2620
- checkpoint: changesCheckpoint,
2621
- maxItems: input.limits.maxItems,
2622
- maxProviderRequests: input.limits.maxProviderRequests,
2623
- maxElapsedMs,
2624
- maxInvocationElapsedMs: Math.min(GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS, maxElapsedMs),
2625
- maxFileBytes: input.limits.maxFileBytes,
2626
- listChanges: async (pageToken, pageSize) => await provider.listChanges(pageToken, input.selectedSource.driveId, pageSize),
2627
- getFile: provider.getFile,
2628
- observedExternalObjectIds: input.observedExternalObjectIds,
2629
- now: clock
2630
- });
2631
- if (changes.status === "paused" && changes.checkpoint) {
2632
- return {
2633
- status: "paused",
2634
- stopReason: changes.stopReason,
2635
- entries: changes.entries,
2636
- checkpoint: {
2637
- version: 2,
2638
- kind: "google_drive_changes",
2639
- ...expectedCursor,
2640
- changesCheckpoint: {
2641
- ...changes.checkpoint,
2642
- requiresFullReconciliation: changes.checkpoint.requiresFullReconciliation || forceFullReconciliation
2643
- }
2644
- },
2645
- providerCursor,
2646
- authoritativeFullScan: false,
2647
- cursorInvalidated: false,
2648
- providerRequests: changes.providerRequests,
2649
- elapsedMs: changes.elapsedMs,
2650
- hardLimitReached: changes.hardLimitReached
2651
- };
2652
- }
2653
- if (!changes.newStartPageToken) throw new Error("google_drive_changes_cursor_missing");
2654
- if (changes.requiresFullReconciliation || forceFullReconciliation) {
2655
- const hardStopReason = googleDriveHardLimitReason(changes.budget, budgetLimits);
2656
- return {
2657
- status: "paused",
2658
- stopReason: hardStopReason ?? "elapsed_time_limit",
2659
- entries: changes.entries,
2660
- checkpoint: buildGoogleDriveFullReconciliationCheckpoint({
2661
- ...expectedCursor,
2662
- boundaryId: cursor.boundaryId,
2663
- startPageToken: changes.newStartPageToken,
2664
- cursorInvalidated: false,
2665
- budgetBeforeInventory: changes.budget,
2666
- inventoryElapsedMs: 0,
2667
- inventoryCheckpoint: null,
2668
- revisionFloors: revisionFloorsForEntries(changes.entries, input.limits.maxItems)
2669
- }),
2670
- providerCursor,
2671
- authoritativeFullScan: false,
2672
- cursorInvalidated: false,
2673
- providerRequests: changes.providerRequests,
2674
- elapsedMs: changes.elapsedMs,
2675
- hardLimitReached: hardStopReason !== null
2676
- };
2677
- }
2678
- return {
2679
- status: "complete",
2680
- stopReason: null,
2681
- entries: changes.entries,
2682
- checkpoint: null,
2683
- providerCursor: advanceGoogleDriveChangesCursor(cursor, changes.newStartPageToken),
2684
- authoritativeFullScan: false,
2685
- cursorInvalidated: false,
2686
- providerRequests: changes.providerRequests,
2687
- elapsedMs: changes.elapsedMs,
2688
- hardLimitReached: false
2689
- };
2690
- } catch (error) {
2691
- if (error instanceof GoogleDriveChangesProtocolError) {
2692
- throw new SyncFailure("provider_payload_invalid", false);
2693
- }
2694
- if (!(error instanceof GoogleDriveCursorInvalidError)) throw error;
2695
- const full = await acquireFullReconciliation({
2696
- budget: error.budget ?? emptyGoogleDriveSyncBudget(),
2697
- boundaryId: cursor.boundaryId,
2698
- cursorInvalidated: true
2699
- });
2700
- const hardStopReason = googleDriveHardLimitReason(full.budgetBeforeInventory, budgetLimits);
2701
- return {
2702
- status: "paused",
2703
- stopReason: hardStopReason ?? "elapsed_time_limit",
2704
- entries: [],
2705
- checkpoint: buildGoogleDriveFullReconciliationCheckpoint({
2706
- ...expectedCursor,
2707
- ...full
2708
- }),
2709
- providerCursor,
2710
- authoritativeFullScan: false,
2711
- cursorInvalidated: true,
2712
- providerRequests: full.budgetBeforeInventory.providerRequests,
2713
- elapsedMs: full.budgetBeforeInventory.elapsedMs,
2714
- hardLimitReached: hardStopReason !== null
2715
- };
2716
- }
2717
- },
2718
- fetchContent: async (entry, maxBytes) => await fetchDriveBytes(entry, input.authorization, maxBytes, input.retry, input.observability),
2719
- readAcl: async (entry, maxProviderRequests) => {
2720
- if (!provider.listPermissions) {
2721
- throw new SyncFailure("provider_rejected", false);
2722
- }
2723
- return await collectGoogleDriveAclEvidence({
2724
- entry,
2725
- maxProviderRequests,
2726
- listPermissions: provider.listPermissions,
2727
- getFile: provider.getFile,
2728
- now: clock
2729
- });
2730
- },
2731
- citationLocator: (entry) => ({
2732
- version: 2,
2733
- providerKey: "google-drive",
2734
- providerCoordinationKey: input.actionProviderCoordinationKey,
2735
- externalObjectId: entry.externalObjectId,
2736
- providerRevision: entry.externalVersionId,
2737
- driveId: entry.driveId,
2738
- sourceUri: entry.sourceUri
2739
- })
2740
- };
2741
- }
2742
- function isLegacyGoogleDriveFullReconciliationCheckpoint(value, expected) {
2743
- return value?.version === 2 && value.kind === "google_drive_full_reconciliation" && value.connectionId === expected.connectionId && value.googlePermissionId === expected.googlePermissionId && value.sourceId === expected.sourceId && value.driveId === expected.driveId && encodedJsonBytes(value) <= GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES;
2744
- }
2745
- function parseGoogleDriveExecutionCheckpoint(value, expected, limits) {
2746
- if (!value || value.connectionId !== expected.connectionId || value.googlePermissionId !== expected.googlePermissionId || value.sourceId !== expected.sourceId || value.driveId !== expected.driveId || encodedJsonBytes(value) > GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES) {
2747
- return null;
2748
- }
2749
- if (value.version === 2 && value.kind === "google_drive_changes") {
2750
- const changesCheckpoint = parseGoogleDriveChangesCheckpoint(value.changesCheckpoint, limits);
2751
- return changesCheckpoint ? { ...value, changesCheckpoint } : null;
2752
- }
2753
- if (value.version === 3 && value.kind === "google_drive_full_reconciliation") {
2754
- const budgetBeforeInventory = parseGoogleDriveSyncBudget(value.budgetBeforeInventory, limits);
2755
- const revisionFloors = parseGoogleDriveRevisionFloors(value.revisionFloors, limits.maxItems);
2756
- if (typeof value.boundaryId !== "string" || value.boundaryId.length < 1 || value.boundaryId.length > 1024 || typeof value.startPageToken !== "string" || value.startPageToken.length < 1 || value.startPageToken.length > 4096 || typeof value.cursorInvalidated !== "boolean" || !budgetBeforeInventory || !revisionFloors || !Number.isSafeInteger(value.inventoryElapsedMs) || Number(value.inventoryElapsedMs) < 0 || Number(value.inventoryElapsedMs) > limits.maxElapsedMs || budgetBeforeInventory.elapsedMs + Number(value.inventoryElapsedMs) > limits.maxElapsedMs || value.inventoryCheckpoint !== null && (!value.inventoryCheckpoint || typeof value.inventoryCheckpoint !== "object" || Array.isArray(value.inventoryCheckpoint))) {
2757
- return null;
2758
- }
2759
- return {
2760
- ...value,
2761
- budgetBeforeInventory,
2762
- inventoryElapsedMs: Number(value.inventoryElapsedMs),
2763
- revisionFloors
2764
- };
2765
- }
2766
- return null;
2767
- }
2768
- function buildGoogleDriveFullReconciliationCheckpoint(input) {
2769
- const checkpoint = {
2770
- version: 3,
2771
- kind: "google_drive_full_reconciliation",
2772
- ...input,
2773
- revisionFloors: input.revisionFloors.map((floor) => [...floor])
2774
- };
2775
- if (encodedJsonBytes(checkpoint) > GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES) {
2776
- throw new SyncFailure("resource_limit", false);
2777
- }
2778
- return checkpoint;
2779
- }
2780
- function parseGoogleDriveRevisionFloors(value, maxItems) {
2781
- if (!Array.isArray(value) || value.length > maxItems) return null;
2782
- const floors = [];
2783
- const seen = /* @__PURE__ */ new Set();
2784
- for (const floor of value) {
2785
- if (!Array.isArray(floor) || floor.length !== 2 || typeof floor[0] !== "string" || floor[0].trim().length < 1 || floor[0].length > 1024 || floor[1] !== null && (typeof floor[1] !== "string" || floor[1].length < 1 || floor[1].length > 256) || seen.has(floor[0])) {
2786
- return null;
2787
- }
2788
- seen.add(floor[0]);
2789
- floors.push([floor[0], floor[1]]);
2790
- }
2791
- return floors;
2792
- }
2793
- function revisionFloorsForEntries(entries, maxItems) {
2794
- const reconciled = reconcileGoogleDriveRevisionFloors(entries, [], maxItems);
2795
- if (reconciled.conflict) throw new SyncFailure("provider_payload_invalid", false);
2796
- return reconciled.revisionFloors;
2797
- }
2798
- function shouldProcessGoogleDriveDurableObservation(input) {
2799
- if (input.observation.externalObjectId !== input.entry.externalObjectId) {
2800
- throw new SyncFailure("provider_payload_invalid", false);
2801
- }
2802
- if (input.observation.disposition === "accepted" || input.observation.disposition === "replayed") {
2803
- return true;
2804
- }
2805
- if (input.observation.disposition === "conflict") {
2806
- throw new SyncFailure("provider_payload_invalid", false);
2807
- }
2808
- const current = input.observation.currentVersion;
2809
- const comparison = current ? compareCanonicalDecimalProviderRevisions(
2810
- current.providerRevision,
2811
- input.observation.providerRevision
2812
- ) : null;
2813
- const fullyProcessed = current !== null && current.objectLifecycleState === "active" && current.sourceLifecycleGeneration === input.sourceLifecycleGeneration && current.objectLifecycleGeneration === current.currentObjectLifecycleGeneration && current.aclGeneration === input.aclGeneration && current.indexObligationStatus === "indexed" && comparison !== null && comparison >= 0 && (comparison > 0 || current.metadataHash === input.observation.metadataHash);
2814
- if (!fullyProcessed) throw new SyncFailure("provider_payload_invalid", false);
2815
- return false;
2816
- }
2817
- function mergeGoogleDriveDurableObservationFloors(checkpoint, observations, maxItems) {
2818
- if (!checkpoint || checkpoint.version !== 3 || checkpoint.kind !== "google_drive_full_reconciliation") {
2819
- return checkpoint;
2820
- }
2821
- const parsedFloors = parseGoogleDriveRevisionFloors(checkpoint.revisionFloors, maxItems);
2822
- if (!parsedFloors) throw new SyncFailure("provider_payload_invalid", false);
2823
- const floors = new Map(parsedFloors);
2824
- for (const observation of observations) {
2825
- floors.set(observation.externalObjectId, observation.providerRevision);
2826
- }
2827
- if (floors.size > maxItems) throw new SyncFailure("resource_limit", false);
2828
- const merged = {
2829
- ...checkpoint,
2830
- revisionFloors: [...floors]
2831
- };
2832
- if (encodedJsonBytes(merged) > GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES) {
2833
- throw new SyncFailure("resource_limit", false);
2834
- }
2835
- return merged;
2836
- }
2837
- function reconcileGoogleDriveRevisionFloors(entries, existingFloors, maxItems) {
2838
- const floors = new Map(existingFloors);
2839
- const accepted = [];
2840
- for (const entry of entries) {
2841
- if (!floors.has(entry.externalObjectId)) {
2842
- floors.set(entry.externalObjectId, entry.externalVersionId);
2843
- accepted.push(entry);
2844
- } else {
2845
- const comparison = compareCanonicalDecimalProviderRevisions(
2846
- entry.externalVersionId,
2847
- floors.get(entry.externalObjectId) ?? null
2848
- );
2849
- if (comparison === null) {
2850
- return {
2851
- entries: [],
2852
- revisionFloors: [...floors],
2853
- conflict: true
2854
- };
2855
- }
2856
- if (comparison > 0) {
2857
- floors.set(entry.externalObjectId, entry.externalVersionId);
2858
- accepted.push(entry);
2859
- }
2860
- }
2861
- if (floors.size > maxItems) throw new SyncFailure("resource_limit", false);
2862
- }
2863
- return {
2864
- entries: accepted,
2865
- revisionFloors: [...floors],
2866
- conflict: false
2867
- };
2868
- }
2869
- function encodedJsonBytes(value) {
2870
- try {
2871
- return new TextEncoder().encode(JSON.stringify(value)).byteLength;
2872
- } catch {
2873
- return Number.POSITIVE_INFINITY;
2874
- }
2875
- }
2876
- function parseGoogleDriveSyncBudget(value, limits) {
2877
- if (!value || typeof value !== "object" || Array.isArray(value)) return null;
2878
- const row = value;
2879
- for (const [key, maximum] of [
2880
- ["examinedItems", limits.maxItems],
2881
- ["providerRequests", limits.maxProviderRequests],
2882
- ["elapsedMs", limits.maxElapsedMs]
2883
- ]) {
2884
- const budgetValue = row[key];
2885
- if (!Number.isSafeInteger(budgetValue) || Number(budgetValue) < 0 || Number(budgetValue) > maximum) {
2886
- return null;
2887
- }
2888
- }
2889
- return value;
2890
- }
2891
- function emptyGoogleDriveSyncBudget() {
2892
- return { examinedItems: 0, providerRequests: 0, elapsedMs: 0 };
2893
- }
2894
- function googleDriveHardLimitReason(budget, limits) {
2895
- if (budget.examinedItems >= limits.maxItems) return "item_limit";
2896
- if (budget.providerRequests >= limits.maxProviderRequests) return "api_request_limit";
2897
- if (budget.elapsedMs >= limits.maxElapsedMs) return "elapsed_time_limit";
2898
- return null;
2899
- }
2900
- function googleDriveInventoryStopIsHard(reason, budget, limits) {
2901
- if (reason === "api_request_limit" || reason === "item_limit" || reason === "known_byte_limit") {
2902
- return true;
2903
- }
2904
- return reason === "elapsed_time_limit" && budget.elapsedMs >= limits.maxElapsedMs;
2905
- }
2906
- async function getDriveStartPageToken(driveId, authorization, retry, observability) {
2907
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2908
- const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
2909
- url.searchParams.set("supportsAllDrives", "true");
2910
- if (driveId) url.searchParams.set("driveId", driveId);
2911
- const response = await requestGoogleDrive(
2912
- url,
2913
- { headers: { authorization, accept: "application/json" } },
2914
- "start_page_token",
2915
- retry,
2916
- observability
2917
- );
2918
- if (!response.ok) throwGoogleDriveResponse(response);
2919
- const payload = await readResponseJsonBounded(
2920
- response,
2921
- DRIVE_JSON_MAX_BYTES,
2922
- "Google Drive start page token"
2923
- );
2924
- return requiredString(payload.startPageToken, "start_page_token");
2925
- }
2926
- async function listDriveChanges(pageToken, driveId, pageSize, authorization, retry, observability) {
2927
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2928
- const url = new URL(`${DRIVE_API_BASE}/changes`);
2929
- url.searchParams.set("pageToken", pageToken);
2930
- url.searchParams.set("pageSize", String(pageSize));
2931
- url.searchParams.set("supportsAllDrives", "true");
2932
- url.searchParams.set("includeItemsFromAllDrives", "true");
2933
- url.searchParams.set("spaces", "drive");
2934
- url.searchParams.set(
2935
- "fields",
2936
- "nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))"
2937
- );
2938
- if (driveId) url.searchParams.set("driveId", driveId);
2939
- const response = await requestGoogleDrive(
2940
- url,
2941
- { headers: { authorization, accept: "application/json" } },
2942
- "list_changes",
2943
- retry,
2944
- observability
2945
- );
2946
- if (response.status === 410) throw new GoogleDriveCursorInvalidError();
2947
- if (!response.ok) throwGoogleDriveResponse(response);
2948
- const payload = await readResponseJsonBounded(
2949
- response,
2950
- DRIVE_JSON_MAX_BYTES,
2951
- "Google Drive changes list"
2952
- );
2953
- const changes = Array.isArray(payload.changes) ? payload.changes : [];
2954
- return {
2955
- changes: changes.map((value) => {
2956
- const row = value && typeof value === "object" ? value : {};
2957
- const fileId = requiredString(row.fileId, "change_file_id");
2958
- return {
2959
- fileId,
2960
- removed: row.removed === true,
2961
- file: row.file && typeof row.file === "object" ? parseDriveProviderItem(row.file) : null
2962
- };
2963
- }),
2964
- nextPageToken: optionalString(payload.nextPageToken),
2965
- newStartPageToken: optionalString(payload.newStartPageToken)
2966
- };
2967
- }
2968
- async function getDriveFile(fileId, authorization, retry, observability) {
2969
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2970
- const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
2971
- url.searchParams.set("supportsAllDrives", "true");
2972
- url.searchParams.set(
2973
- "fields",
2974
- "id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed"
2975
- );
2976
- const response = await requestGoogleDrive(
2977
- url,
2978
- { headers: { authorization, accept: "application/json" } },
2979
- "get_file",
2980
- retry,
2981
- observability
2982
- );
2983
- if (response.status === 404) return null;
2984
- if (!response.ok) throwGoogleDriveResponse(response);
2985
- return parseDriveProviderItem(
2986
- await readResponseJsonBounded(response, DRIVE_JSON_MAX_BYTES, "Google Drive file metadata")
2987
- );
2988
- }
2989
- async function listDrivePermissions(input, authorization, retry, observability) {
2990
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
2991
- const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(input.fileId)}/permissions`);
2992
- url.searchParams.set("supportsAllDrives", "true");
2993
- url.searchParams.set("pageSize", String(input.pageSize));
2994
- url.searchParams.set(
2995
- "fields",
2996
- "nextPageToken,permissions(id,type,emailAddress,domain,role,allowFileDiscovery,expirationTime,deleted,permissionDetails(inherited))"
2997
- );
2998
- if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
2999
- const response = await requestGoogleDrive(
3000
- url,
3001
- { headers: { authorization, accept: "application/json" } },
3002
- "list_permissions",
3003
- retry,
3004
- observability
3005
- );
3006
- if (response.status === 403 || response.status === 404) {
3007
- return { permissions: [], nextPageToken: null, denied: true };
3008
- }
3009
- if (!response.ok) throwGoogleDriveResponse(response);
3010
- const payload = await readResponseJsonBounded(
3011
- response,
3012
- DRIVE_JSON_MAX_BYTES,
3013
- "Google Drive permission list"
3014
- );
3015
- const rawPermissions = Array.isArray(payload.permissions) ? payload.permissions : null;
3016
- if (!rawPermissions) throw new SyncFailure("provider_payload_invalid", false);
3017
- return {
3018
- permissions: rawPermissions.map(parseDrivePermission),
3019
- nextPageToken: optionalString(payload.nextPageToken),
3020
- denied: false
3021
- };
3022
- }
3023
- async function collectGoogleDriveAclEvidence(input) {
3024
- if (!Number.isSafeInteger(input.maxProviderRequests) || input.maxProviderRequests < 1) {
3025
- throw new SyncFailure("resource_limit", false);
3026
- }
3027
- const now = input.now ?? Date.now;
3028
- const observedAt = new Date(now());
3029
- if (!Number.isFinite(observedAt.getTime())) {
3030
- throw new SyncFailure("provider_payload_invalid", false);
3031
- }
3032
- const permissions = [];
3033
- const seenTokens = /* @__PURE__ */ new Set();
3034
- let pageToken = null;
3035
- let providerRequests = 0;
3036
- let permissionListDenied = false;
3037
- do {
3038
- if (providerRequests >= input.maxProviderRequests) {
3039
- throw new SyncFailure("resource_limit", false);
3040
- }
3041
- providerRequests += 1;
3042
- const page = await input.listPermissions({
3043
- fileId: input.entry.externalObjectId,
3044
- pageToken,
3045
- pageSize: Math.min(100, GOOGLE_DRIVE_ACL_MAX_PRINCIPALS - permissions.length)
3046
- });
3047
- if (page.denied) {
3048
- permissionListDenied = true;
3049
- break;
3050
- }
3051
- permissions.push(...page.permissions);
3052
- if (permissions.length > GOOGLE_DRIVE_ACL_MAX_PRINCIPALS) {
3053
- throw new SyncFailure("resource_limit", false);
3054
- }
3055
- pageToken = page.nextPageToken;
3056
- if (pageToken) {
3057
- if (pageToken.length > 4096 || seenTokens.has(pageToken)) {
3058
- throw new SyncFailure("provider_payload_invalid", false);
3059
- }
3060
- seenTokens.add(pageToken);
3061
- }
3062
- } while (pageToken);
3063
- if (permissionListDenied) {
3064
- if (!input.getFile || providerRequests >= input.maxProviderRequests) {
3065
- return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
3066
- }
3067
- providerRequests += 1;
3068
- const current = await input.getFile(input.entry.externalObjectId);
3069
- if (!current || current.trashed) {
3070
- return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
3071
- }
3072
- const aclRevision2 = createHash2("sha256").update(
3073
- JSON.stringify({
3074
- version: 1,
3075
- mode: "source_owner_only",
3076
- externalObjectId: input.entry.externalObjectId,
3077
- providerRevision: input.entry.externalVersionId
3078
- })
3079
- ).digest("hex");
3080
- return {
3081
- eligibility: "eligible",
3082
- providerRevision: input.entry.externalVersionId,
3083
- driveId: input.entry.driveId,
3084
- aclRevision: aclRevision2,
3085
- observedAt: observedAt.toISOString(),
3086
- expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
3087
- providerRequests,
3088
- principals: []
3089
- };
3090
- }
3091
- const canonicalPermissions = permissions.map((permission) => ({
3092
- id: permission.id,
3093
- type: permission.type,
3094
- role: permission.role,
3095
- emailAddress: permission.emailAddress,
3096
- domain: permission.domain,
3097
- allowFileDiscovery: permission.allowFileDiscovery,
3098
- expirationTime: permission.expirationTime,
3099
- inherited: permission.inherited
3100
- })).sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
3101
- const aclRevision = createHash2("sha256").update(
3102
- JSON.stringify({
3103
- version: 1,
3104
- externalObjectId: input.entry.externalObjectId,
3105
- providerRevision: input.entry.externalVersionId,
3106
- permissions: canonicalPermissions
3107
- })
3108
- ).digest("hex");
3109
- return {
3110
- eligibility: "eligible",
3111
- providerRevision: input.entry.externalVersionId,
3112
- driveId: input.entry.driveId,
3113
- aclRevision,
3114
- observedAt: observedAt.toISOString(),
3115
- expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
3116
- providerRequests,
3117
- principals: canonicalPermissions.map((permission) => ({
3118
- type: permission.type,
3119
- permissionId: permission.id,
3120
- emailAddress: permission.emailAddress,
3121
- domain: permission.domain,
3122
- role: permission.role,
3123
- inherited: permission.inherited,
3124
- allowFileDiscovery: permission.allowFileDiscovery,
3125
- expirationTime: permission.expirationTime
3126
- }))
3127
- };
3128
- }
3129
- function googleDriveDeniedAclEvidence(entry, observedAt, providerRequests) {
3130
- return {
3131
- eligibility: "denied",
3132
- providerRevision: entry.externalVersionId,
3133
- driveId: entry.driveId,
3134
- aclRevision: createHash2("sha256").update(
3135
- JSON.stringify({
3136
- version: 1,
3137
- mode: "provider_denied",
3138
- externalObjectId: entry.externalObjectId,
3139
- providerRevision: entry.externalVersionId
3140
- })
3141
- ).digest("hex"),
3142
- observedAt: observedAt.toISOString(),
3143
- expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
3144
- providerRequests,
3145
- principals: []
3146
- };
3147
- }
3148
- function parseDrivePermission(value) {
3149
- const row = objectRecord(value);
3150
- if (row.deleted === true) throw new SyncFailure("provider_payload_invalid", false);
3151
- const type = requiredString(row.type, "permission.type");
3152
- if (type !== "user" && type !== "group" && type !== "domain" && type !== "anyone") {
3153
- throw new SyncFailure("provider_payload_invalid", false);
3154
- }
3155
- const role = requiredString(row.role, "permission.role");
3156
- if (role !== "owner" && role !== "organizer" && role !== "fileOrganizer" && role !== "writer" && role !== "commenter" && role !== "reader") {
3157
- throw new SyncFailure("provider_payload_invalid", false);
3158
- }
3159
- const permissionDetails = Array.isArray(row.permissionDetails) ? row.permissionDetails : [];
3160
- return {
3161
- id: requiredString(row.id, "permission.id"),
3162
- type,
3163
- role,
3164
- emailAddress: optionalString(row.emailAddress),
3165
- domain: optionalString(row.domain),
3166
- allowFileDiscovery: typeof row.allowFileDiscovery === "boolean" ? row.allowFileDiscovery : null,
3167
- expirationTime: optionalExactProviderTimestamp(row.expirationTime),
3168
- inherited: permissionDetails.some((detail) => objectRecord(detail).inherited === true)
3169
- };
3170
- }
3171
- function optionalExactProviderTimestamp(value) {
3172
- const timestamp = optionalString(value);
3173
- if (!timestamp) return null;
3174
- const parsed = new Date(timestamp);
3175
- if (!Number.isFinite(parsed.getTime())) throw new SyncFailure("provider_payload_invalid", false);
3176
- return parsed.toISOString();
3177
- }
3178
- function throwGoogleDriveResponse(response) {
3179
- if (response.status === 401 || response.status === 403) {
3180
- throw new SyncFailure("connection_reconnect_required", true, true);
3181
- }
3182
- throw new SyncFailure(
3183
- response.status >= 500 || response.status === 429 ? "provider_unavailable" : "provider_rejected",
3184
- response.status >= 500 || response.status === 429
3185
- );
3186
- }
3187
- async function requestGoogleDrive(url, init, operation, retry, observability) {
3188
- try {
3189
- return await fetchGoogleDriveProvider({
3190
- fetchImpl: fetch,
3191
- url,
3192
- init,
3193
- operation,
3194
- policy: retry,
3195
- observability,
3196
- heartbeat: (details) => heartbeat({ provider: "google_drive", ...details })
3197
- });
3198
- } catch (error) {
3199
- if (error instanceof GoogleDriveProviderTransportError) {
3200
- throw new SyncFailure("provider_unavailable", true);
3201
- }
3202
- throw error;
3203
- }
3204
- }
3205
- function selectedDriveSource(metadata, externalSourceId) {
3206
- const sources = metadata.selectedSources ?? (metadata.selectedSource ? [metadata.selectedSource] : []);
3207
- return sources.find((source) => source.id === externalSourceId) ?? null;
3208
- }
3209
- function googleDriveSyncProviderAccessAllowed(input) {
3210
- return input.resolvedConnectionVersion !== void 0 && input.connectionVersion === input.resolvedConnectionVersion && input.connectionStatus === "active" && (input.lifecycleState === void 0 || input.lifecycleState === "active") && googleDriveScopesAllowCapability(input.grantedScopes, "recursive_source_sync");
3211
- }
3212
- async function listDriveChildren(input, authorization, retry, observability) {
3213
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
3214
- const url = new URL(`${DRIVE_API_BASE}/files`);
3215
- url.searchParams.set(
3216
- "q",
3217
- `'${input.folderId.replaceAll("'", "\\'")}' in parents and trashed = false`
3218
- );
3219
- url.searchParams.set("pageSize", String(input.pageSize));
3220
- url.searchParams.set("supportsAllDrives", "true");
3221
- url.searchParams.set("includeItemsFromAllDrives", "true");
3222
- url.searchParams.set(
3223
- "fields",
3224
- "nextPageToken,incompleteSearch,files(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed)"
3225
- );
3226
- if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
3227
- if (input.driveId) {
3228
- url.searchParams.set("corpora", "drive");
3229
- url.searchParams.set("driveId", input.driveId);
3230
- }
3231
- const response = await requestGoogleDrive(
3232
- url,
3233
- { headers: { authorization, accept: "application/json" } },
3234
- "list_children",
3235
- retry,
3236
- observability
3237
- );
3238
- if (!response.ok) {
3239
- if (response.status === 401 || response.status === 403) {
3240
- throw new SyncFailure("connection_reconnect_required", true, true);
3241
- }
3242
- throw new SyncFailure(
3243
- "provider_unavailable",
3244
- response.status >= 500 || response.status === 429
3245
- );
3246
- }
3247
- const payload = await readResponseJsonBounded(
3248
- response,
3249
- DRIVE_JSON_MAX_BYTES,
3250
- "Google Drive file list"
3251
- );
3252
- const rawFiles = Array.isArray(payload.files) ? payload.files : [];
3253
- return {
3254
- items: rawFiles.map(parseDriveProviderItem),
3255
- nextPageToken: typeof payload.nextPageToken === "string" ? payload.nextPageToken : null,
3256
- incompleteSearch: payload.incompleteSearch === true
3257
- };
3258
- }
3259
- function parseDriveProviderItem(value) {
3260
- const row = value && typeof value === "object" ? value : {};
3261
- return {
3262
- id: requiredString(row.id, "id"),
3263
- name: requiredString(row.name, "name"),
3264
- mimeType: requiredString(row.mimeType, "mimeType"),
3265
- driveId: optionalString(row.driveId),
3266
- parents: Array.isArray(row.parents) ? row.parents.filter((item) => typeof item === "string") : [],
3267
- modifiedTime: optionalString(row.modifiedTime),
3268
- createdTime: optionalString(row.createdTime),
3269
- version: optionalString(row.version),
3270
- md5Checksum: optionalString(row.md5Checksum),
3271
- size: optionalString(row.size),
3272
- webViewLink: optionalString(row.webViewLink),
3273
- trashed: row.trashed === true
3274
- };
3275
- }
3276
- async function fetchDriveBytes(entry, authorization, maxBytes, retry, observability) {
3277
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
3278
- const url = entry.transfer.action === "export" ? new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}/export`) : new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(entry.externalObjectId)}`);
3279
- if (entry.transfer.action === "export") {
3280
- url.searchParams.set("mimeType", entry.transfer.contentType);
3281
- } else {
3282
- url.searchParams.set("alt", "media");
3283
- url.searchParams.set("supportsAllDrives", "true");
3284
- }
3285
- const response = await requestGoogleDrive(
3286
- url,
3287
- { headers: { authorization } },
3288
- entry.transfer.action === "export" ? "export" : "download",
3289
- retry,
3290
- observability
3291
- );
3292
- if (!response.ok) {
3293
- if (response.status === 401 || response.status === 403) {
3294
- throw new SyncFailure("connection_reconnect_required", true, true);
3295
- }
3296
- throw new SyncFailure(
3297
- "provider_unavailable",
3298
- response.status >= 500 || response.status === 429
3299
- );
3300
- }
3301
- const declared = Number(response.headers.get("content-length") ?? "0");
3302
- if (Number.isFinite(declared) && declared > maxBytes) {
3303
- throw new SyncFailure("content_too_large", false);
3304
- }
3305
- if (!response.body) {
3306
- const bytes2 = new Uint8Array(await response.arrayBuffer());
3307
- if (bytes2.byteLength > maxBytes) throw new SyncFailure("content_too_large", false);
3308
- return bytes2;
3309
- }
3310
- const reader = response.body.getReader();
3311
- const chunks = [];
3312
- let total = 0;
3313
- while (true) {
3314
- const { done, value } = await reader.read();
3315
- if (done) break;
3316
- total += value.byteLength;
3317
- if (total > maxBytes) {
3318
- await reader.cancel();
3319
- throw new SyncFailure("content_too_large", false);
3320
- }
3321
- chunks.push(value);
3322
- }
3323
- const bytes = new Uint8Array(total);
3324
- let offset = 0;
3325
- for (const chunk of chunks) {
3326
- bytes.set(chunk, offset);
3327
- offset += chunk.byteLength;
3328
- }
3329
- return bytes;
3330
- }
3331
- async function listAtlassianItems(source, cursor, pageSize, authorization) {
3332
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
3333
- if (source.kind === "jira_project") {
3334
- const url2 = new URL(
3335
- `${ATLASSIAN_API_BASE}/ex/jira/${encodeURIComponent(source.cloudId)}/rest/api/3/search/jql`
3336
- );
3337
- url2.searchParams.set(
3338
- "jql",
3339
- `project = "${source.key.replaceAll('"', '\\"')}" ORDER BY key ASC`
3340
- );
3341
- url2.searchParams.set("maxResults", String(pageSize));
3342
- url2.searchParams.set("fields", "summary,created,updated");
3343
- if (cursor) url2.searchParams.set("nextPageToken", cursor);
3344
- const payload2 = objectRecord(await atlassianJsonRequest(url2, authorization, "Jira issue list"));
3345
- const issues = Array.isArray(payload2.issues) ? payload2.issues : [];
3346
- return {
3347
- items: issues.map((raw) => {
3348
- const row = objectRecord(raw);
3349
- const fields = objectRecord(row.fields);
3350
- const id = requiredAtlassianString(row.id, "issue.id");
3351
- const key = requiredAtlassianString(row.key, "issue.key");
3352
- const updatedAt = optionalString(fields.updated);
3353
- return {
3354
- id,
3355
- key,
3356
- title: requiredAtlassianString(fields.summary, "issue.summary"),
3357
- version: updatedAt,
3358
- createdAt: optionalString(fields.created),
3359
- updatedAt,
3360
- webUrl: new URL(`/browse/${encodeURIComponent(key)}`, source.siteUrl).toString()
3361
- };
3362
- }),
3363
- nextCursor: optionalString(payload2.nextPageToken)
3364
- };
3365
- }
3366
- const url = new URL(
3367
- `${ATLASSIAN_API_BASE}/ex/confluence/${encodeURIComponent(source.cloudId)}/wiki/api/v2/spaces/${encodeURIComponent(source.resourceId)}/pages`
3368
- );
3369
- url.searchParams.set("limit", String(pageSize));
3370
- url.searchParams.set("sort", "id");
3371
- if (cursor) url.searchParams.set("cursor", cursor);
3372
- const payload = objectRecord(
3373
- await atlassianJsonRequest(url, authorization, "Confluence page list")
3374
- );
3375
- const pages = Array.isArray(payload.results) ? payload.results : [];
3376
- return {
3377
- items: pages.map((raw) => {
3378
- const row = objectRecord(raw);
3379
- const version = objectRecord(row.version);
3380
- const id = requiredAtlassianString(row.id, "page.id");
3381
- const webUi = optionalString(objectRecord(row._links).webui);
3382
- const versionNumber = numberValue(version.number);
3383
- return {
3384
- id,
3385
- key: id,
3386
- title: requiredAtlassianString(row.title, "page.title"),
3387
- version: versionNumber === void 0 ? null : String(versionNumber),
3388
- createdAt: optionalString(row.createdAt),
3389
- updatedAt: optionalString(version.createdAt),
3390
- webUrl: webUi ? new URL(webUi.startsWith("/wiki/") ? webUi : `/wiki${webUi}`, source.siteUrl).toString() : new URL(
3391
- `/wiki/spaces/${encodeURIComponent(source.key)}/pages/${encodeURIComponent(id)}`,
3392
- source.siteUrl
3393
- ).toString()
3394
- };
3395
- }),
3396
- nextCursor: confluenceCursor(objectRecord(payload._links).next)
546
+ }
3397
547
  };
3398
548
  }
3399
- async function fetchAtlassianContent(source, entry, authorization, maxBytes) {
3400
- if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
3401
- const objectId = entry.externalObjectId.slice(entry.externalObjectId.indexOf(":") + 1);
3402
- const markdown = source.kind === "jira_project" ? await jiraIssueMarkdown(source, objectId, entry, authorization) : await confluencePageMarkdown(source, objectId, entry, authorization);
3403
- const bytes = new TextEncoder().encode(markdown);
3404
- if (bytes.byteLength > maxBytes) throw new SyncFailure("content_too_large", false);
3405
- return bytes;
3406
- }
3407
- async function jiraIssueMarkdown(source, issueId, entry, authorization) {
3408
- const issueUrl = new URL(
3409
- `${ATLASSIAN_API_BASE}/ex/jira/${encodeURIComponent(source.cloudId)}/rest/api/3/issue/${encodeURIComponent(issueId)}`
3410
- );
3411
- issueUrl.searchParams.set(
3412
- "fields",
3413
- "summary,description,status,issuetype,priority,assignee,reporter,labels,created,updated"
3414
- );
3415
- const issue = objectRecord(await atlassianJsonRequest(issueUrl, authorization, "Jira issue"));
3416
- const fields = objectRecord(issue.fields);
3417
- const lines = [
3418
- `# ${entry.title}`,
3419
- "",
3420
- `- Key: ${optionalString(issue.key) ?? entry.title}`,
3421
- `- Type: ${optionalString(objectRecord(fields.issuetype).name) ?? "Unknown"}`,
3422
- `- Status: ${optionalString(objectRecord(fields.status).name) ?? "Unknown"}`,
3423
- `- Priority: ${optionalString(objectRecord(fields.priority).name) ?? "None"}`,
3424
- `- Assignee: ${optionalString(objectRecord(fields.assignee).displayName) ?? "Unassigned"}`,
3425
- `- Reporter: ${optionalString(objectRecord(fields.reporter).displayName) ?? "Unknown"}`,
3426
- `- Updated: ${optionalString(fields.updated) ?? "Unknown"}`,
3427
- `- Source: ${entry.sourceUri}`,
3428
- "",
3429
- "## Description",
3430
- "",
3431
- adfToMarkdown(fields.description) || "No description."
3432
- ];
3433
- const labels = Array.isArray(fields.labels) ? fields.labels.filter((value) => typeof value === "string") : [];
3434
- if (labels.length > 0) lines.splice(9, 0, `- Labels: ${labels.join(", ")}`);
3435
- const comments = await jiraComments(source, issueId, authorization);
3436
- if (comments.length > 0) {
3437
- lines.push("", "## Comments", "");
3438
- for (const comment of comments) {
3439
- const row = objectRecord(comment);
3440
- const author = optionalString(objectRecord(row.author).displayName) ?? "Unknown author";
3441
- const created = optionalString(row.created) ?? "Unknown date";
3442
- lines.push(`### ${author} \xB7 ${created}`, "", adfToMarkdown(row.body), "");
3443
- }
3444
- }
3445
- return lines.join("\n");
3446
- }
3447
- async function jiraComments(source, issueId, authorization) {
3448
- const comments = [];
3449
- let startAt = 0;
3450
- for (let page = 0; page < 20; page += 1) {
3451
- const url = new URL(
3452
- `${ATLASSIAN_API_BASE}/ex/jira/${encodeURIComponent(source.cloudId)}/rest/api/3/issue/${encodeURIComponent(issueId)}/comment`
3453
- );
3454
- url.searchParams.set("startAt", String(startAt));
3455
- url.searchParams.set("maxResults", "100");
3456
- url.searchParams.set("orderBy", "created");
3457
- const payload = objectRecord(await atlassianJsonRequest(url, authorization, "Jira comments"));
3458
- const pageComments = Array.isArray(payload.comments) ? payload.comments : [];
3459
- comments.push(...pageComments);
3460
- const total = numberValue(payload.total) ?? comments.length;
3461
- if (comments.length >= total || pageComments.length === 0) break;
3462
- startAt += pageComments.length;
3463
- }
3464
- return comments;
549
+ function isUnprocessableEntity(error) {
550
+ return error instanceof Error && "status" in error && error.status === 422;
3465
551
  }
3466
- async function confluencePageMarkdown(source, pageId, entry, authorization) {
3467
- const pageUrl = new URL(
3468
- `${ATLASSIAN_API_BASE}/ex/confluence/${encodeURIComponent(source.cloudId)}/wiki/api/v2/pages/${encodeURIComponent(pageId)}`
3469
- );
3470
- pageUrl.searchParams.set("body-format", "storage");
3471
- const page = objectRecord(await atlassianJsonRequest(pageUrl, authorization, "Confluence page"));
3472
- const storage = objectRecord(objectRecord(page.body).storage);
3473
- const lines = [
3474
- `# ${entry.title}`,
3475
- "",
3476
- `- Space: ${source.name} (${source.key})`,
3477
- `- Updated: ${entry.modifiedTime ?? "Unknown"}`,
3478
- `- Source: ${entry.sourceUri}`,
3479
- "",
3480
- "## Content",
3481
- "",
3482
- optionalString(storage.value) ?? "No content."
3483
- ];
3484
- const comments = await confluenceFooterComments(source, pageId, authorization);
3485
- if (comments.length > 0) {
3486
- lines.push("", "## Comments", "");
3487
- for (const comment of comments) {
3488
- const row = objectRecord(comment);
3489
- const body = objectRecord(objectRecord(row.body).storage);
3490
- const version = objectRecord(row.version);
3491
- lines.push(
3492
- `### Comment \xB7 ${optionalString(version.createdAt) ?? "Unknown date"}`,
3493
- "",
3494
- optionalString(body.value) ?? "",
3495
- ""
3496
- );
552
+
553
+ // src/activities/file-upload-reaper.ts
554
+ import {
555
+ claimDueTranscriptionRecordingObjectCleanup,
556
+ claimExpiredFileUploadCleanup,
557
+ completeDueTranscriptionRecordingObjectCleanup,
558
+ completeExpiredFileUploadCleanup,
559
+ claimVideoGenerationOperations,
560
+ purgeExpiredTranscriptionRecordings
561
+ } from "@opengeni/db";
562
+ import { randomUUID } from "crypto";
563
+ var FILE_UPLOAD_CLEANUP_GRACE_MS = 60 * 60 * 1e3;
564
+ var FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS = 10 * 60 * 1e3;
565
+ var FILE_UPLOAD_CLEANUP_BATCH_SIZE = 100;
566
+ var SAFE_CLEANUP_ERROR_CATEGORIES = /* @__PURE__ */ new Set([
567
+ "aborted",
568
+ "conflict",
569
+ "invalid_response",
570
+ "network",
571
+ "not_found",
572
+ "permission_denied",
573
+ "provider",
574
+ "timeout",
575
+ "unavailable"
576
+ ]);
577
+ function cleanupErrorAttributes(error) {
578
+ let errorCategory = "provider_error";
579
+ let errorStatus;
580
+ if (error && typeof error === "object") {
581
+ const rawCode = error.code;
582
+ if (typeof rawCode === "string" && SAFE_CLEANUP_ERROR_CATEGORIES.has(rawCode)) {
583
+ errorCategory = rawCode;
584
+ } else if (rawCode !== void 0) {
585
+ errorCategory = "unknown";
3497
586
  }
3498
- }
3499
- return lines.join("\n");
3500
- }
3501
- async function confluenceFooterComments(source, pageId, authorization) {
3502
- const comments = [];
3503
- let cursor = null;
3504
- for (let page = 0; page < 20; page += 1) {
3505
- const url = new URL(
3506
- `${ATLASSIAN_API_BASE}/ex/confluence/${encodeURIComponent(source.cloudId)}/wiki/api/v2/pages/${encodeURIComponent(pageId)}/footer-comments`
3507
- );
3508
- url.searchParams.set("body-format", "storage");
3509
- url.searchParams.set("limit", "100");
3510
- if (cursor) url.searchParams.set("cursor", cursor);
3511
- const payload = objectRecord(
3512
- await atlassianJsonRequest(url, authorization, "Confluence page comments")
3513
- );
3514
- const results = Array.isArray(payload.results) ? payload.results : [];
3515
- comments.push(...results);
3516
- cursor = confluenceCursor(objectRecord(payload._links).next);
3517
- if (!cursor) break;
3518
- }
3519
- return comments;
3520
- }
3521
- async function atlassianJsonRequest(url, authorization, label) {
3522
- const response = await fetch(url, {
3523
- headers: { authorization, accept: "application/json" },
3524
- redirect: "error",
3525
- signal: AbortSignal.timeout(2e4)
3526
- });
3527
- if (!response.ok) {
3528
- await response.body?.cancel().catch(() => void 0);
3529
- if (response.status === 401 || response.status === 403) {
3530
- throw new SyncFailure("connection_reconnect_required", true, true);
3531
- }
3532
- throw new SyncFailure(
3533
- "provider_unavailable",
3534
- response.status >= 500 || response.status === 429
587
+ const rawStatus = Number(
588
+ error.status ?? error.statusCode
3535
589
  );
590
+ if (Number.isInteger(rawStatus) && rawStatus >= 100 && rawStatus <= 599) {
591
+ errorStatus = rawStatus;
592
+ }
3536
593
  }
3537
- return await readResponseJsonBounded(response, ATLASSIAN_JSON_MAX_BYTES, label);
3538
- }
3539
- function confluenceCursor(value) {
3540
- const next = optionalString(value);
3541
- if (!next) return null;
3542
- const url = new URL(next, ATLASSIAN_API_BASE);
3543
- const cursor = url.searchParams.get("cursor");
3544
- if (!cursor || cursor.length > 4096 || /[\u0000-\u001f]/.test(cursor)) {
3545
- throw new Error("invalid Confluence cursor");
3546
- }
3547
- return cursor;
594
+ return { errorCategory, ...errorStatus === void 0 ? {} : { errorStatus } };
3548
595
  }
3549
- function adfToMarkdown(value) {
3550
- const node = objectRecord(value);
3551
- const type = optionalString(node.type);
3552
- const children = Array.isArray(node.content) ? node.content.map(adfToMarkdown).join("") : "";
3553
- if (type === "text") {
3554
- let text = typeof node.text === "string" ? node.text : "";
3555
- const marks = Array.isArray(node.marks) ? node.marks : [];
3556
- for (const rawMark of marks) {
3557
- const mark = objectRecord(rawMark);
3558
- const markType = optionalString(mark.type);
3559
- if (markType === "code") text = `\`${text}\``;
3560
- if (markType === "strong") text = `**${text}**`;
3561
- if (markType === "em") text = `*${text}*`;
3562
- if (markType === "strike") text = `~~${text}~~`;
3563
- if (markType === "link") {
3564
- const href = optionalString(objectRecord(mark.attrs).href);
3565
- if (href) text = `[${text}](${href})`;
596
+ function createFileUploadReaperActivities(services, options = {}) {
597
+ const graceMs = options.graceMs ?? FILE_UPLOAD_CLEANUP_GRACE_MS;
598
+ const claimTimeoutMs = options.claimTimeoutMs ?? FILE_UPLOAD_CLEANUP_CLAIM_TIMEOUT_MS;
599
+ const batchSize = options.batchSize ?? FILE_UPLOAD_CLEANUP_BATCH_SIZE;
600
+ const deleteObject = options.deleteObject ?? (async (storage, key) => storage.deleteObject(key));
601
+ const claimFileUploads = options.claimFileUploads ?? claimExpiredFileUploadCleanup;
602
+ const completeFileUpload = options.completeFileUpload ?? completeExpiredFileUploadCleanup;
603
+ const claimTranscriptionObjects = options.claimTranscriptionObjects ?? claimDueTranscriptionRecordingObjectCleanup;
604
+ const completeTranscriptionObject = options.completeTranscriptionObject ?? completeDueTranscriptionRecordingObjectCleanup;
605
+ const purgeTranscriptionRecordings = options.purgeTranscriptionRecordings ?? purgeExpiredTranscriptionRecordings;
606
+ async function reapExpiredFileUploads() {
607
+ const { db, objectStorage, observability } = await services();
608
+ if (!objectStorage) {
609
+ try {
610
+ await purgeTranscriptionRecordings(db, { graceMs, limit: batchSize });
611
+ } catch (error) {
612
+ observability.warn("expired transcription recording metadata purge failed", {
613
+ ...cleanupErrorAttributes(error)
614
+ });
3566
615
  }
616
+ return { claimed: 0, deleted: 0, failed: 0 };
3567
617
  }
3568
- return text;
3569
- }
3570
- if (type === "paragraph") return `${children}
3571
-
3572
- `;
3573
- if (type === "heading") {
3574
- const level = Math.min(Math.max(numberValue(objectRecord(node.attrs).level) ?? 2, 1), 6);
3575
- return `${"#".repeat(level)} ${children.trim()}
3576
-
3577
- `;
3578
- }
3579
- if (type === "hardBreak") return " \n";
3580
- if (type === "rule") return "---\n\n";
3581
- if (type === "blockquote") {
3582
- return `${children.trim().split("\n").map((line) => `> ${line}`).join("\n")}
3583
-
3584
- `;
3585
- }
3586
- if (type === "bulletList") {
3587
- return `${children.trimEnd()}
3588
- `;
3589
- }
3590
- if (type === "orderedList") {
3591
- return `${children.trimEnd()}
3592
- `;
3593
- }
3594
- if (type === "listItem") {
3595
- return `- ${children.trim().replaceAll("\n", "\n ")}
3596
- `;
3597
- }
3598
- if (type === "codeBlock") return `\`\`\`
3599
- ${children.trimEnd()}
3600
- \`\`\`
3601
-
3602
- `;
3603
- if (type === "mention") return `@${optionalString(objectRecord(node.attrs).text) ?? "user"}`;
3604
- if (type === "emoji") return optionalString(objectRecord(node.attrs).text) ?? "";
3605
- if (type === "inlineCard") {
3606
- const url = optionalString(objectRecord(node.attrs).url);
3607
- return url ? `[${url}](${url})` : "";
3608
- }
3609
- return children;
3610
- }
3611
- function objectRecord(value) {
3612
- return value && typeof value === "object" && !Array.isArray(value) ? value : {};
3613
- }
3614
- function requiredAtlassianString(value, label) {
3615
- if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_atlassian_${label}`);
3616
- return value;
3617
- }
3618
- function deterministicUuid(value) {
3619
- const bytes = createHash2("sha256").update(value).digest().subarray(0, 16);
3620
- bytes[6] = bytes[6] & 15 | 80;
3621
- bytes[8] = bytes[8] & 63 | 128;
3622
- const hex = bytes.toString("hex");
3623
- return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
3624
- }
3625
- function safeFilename(value) {
3626
- const normalized = value.normalize("NFKC").replace(/[\\/:*?"<>|\0-\x1f]/g, "_").trim();
3627
- return (normalized || "document").slice(0, 240);
3628
- }
3629
- function requiredString(value, label) {
3630
- if (typeof value !== "string" || !value.trim()) throw new Error(`invalid_google_drive_${label}`);
3631
- return value;
3632
- }
3633
- function optionalString(value) {
3634
- return typeof value === "string" && value.length > 0 ? value : null;
3635
- }
3636
- function numberValue(value) {
3637
- return typeof value === "number" && Number.isFinite(value) ? value : void 0;
3638
- }
3639
- function boundedError(error) {
3640
- return (error instanceof Error ? error.message : String(error)).slice(0, 1e3);
3641
- }
3642
- function entryMetadataHash(entry) {
3643
- return createHash2("sha256").update(
3644
- JSON.stringify({
3645
- externalObjectId: entry.externalObjectId,
3646
- externalVersionId: entry.externalVersionId,
3647
- title: entry.title,
3648
- mimeType: entry.mimeType,
3649
- driveId: entry.driveId,
3650
- modifiedTime: entry.modifiedTime,
3651
- createdTime: entry.createdTime,
3652
- sourceUri: entry.sourceUri
3653
- })
3654
- ).digest("hex");
3655
- }
3656
- function inventoryLimit(reason) {
3657
- if (reason === "api_request_limit") return "provider_requests";
3658
- if (reason === "elapsed_time_limit") return "elapsed_time";
3659
- if (reason === "item_limit") return "items";
3660
- if (reason === "known_byte_limit") return "bytes";
3661
- return null;
3662
- }
3663
- function recordSyncHealthTelemetry(observability, provider, summary, outcome, failure) {
3664
- observability.observeHistogram({
3665
- name: "opengeni_knowledge_source_sync_terminal_batch_duration_seconds",
3666
- help: "Terminal knowledge-source activity batch duration by provider and outcome.",
3667
- labels: { provider, outcome },
3668
- value: Math.max(0, summary.elapsedMs) / 1e3
3669
- });
3670
- if (summary.providerRequests > 0) {
3671
- observability.incrementCounter({
3672
- name: "opengeni_knowledge_source_sync_provider_requests_total",
3673
- help: "Logical provider requests consumed by knowledge-source synchronization.",
3674
- labels: { provider, outcome },
3675
- amount: summary.providerRequests
618
+ const claims = await claimFileUploads(db, {
619
+ graceMs,
620
+ claimTimeoutMs,
621
+ limit: batchSize
3676
622
  });
3677
- }
3678
- if (summary.limitReached) {
3679
- observability.incrementCounter({
3680
- name: "opengeni_knowledge_source_sync_limit_hits_total",
3681
- help: "Knowledge-source synchronization runs stopped by an explicit resource budget.",
3682
- labels: { provider, limit: summary.limitReached }
623
+ let deleted = 0;
624
+ let failed = 0;
625
+ for (const claim of claims) {
626
+ try {
627
+ await deleteObject(objectStorage, claim.objectKey);
628
+ const settled = await completeFileUpload(db, claim);
629
+ if (!settled) {
630
+ throw new Error("cleanup claim no longer owns a reclaimable upload");
631
+ }
632
+ deleted += 1;
633
+ } catch (error) {
634
+ failed += 1;
635
+ observability.warn("expired file upload cleanup failed; claim remains reclaimable", {
636
+ workspaceId: claim.workspaceId,
637
+ uploadId: claim.uploadId,
638
+ fileId: claim.fileId,
639
+ ...cleanupErrorAttributes(error)
640
+ });
641
+ }
642
+ }
643
+ const recordingClaims = await claimTranscriptionObjects(db, {
644
+ graceMs,
645
+ claimTimeoutMs,
646
+ limit: batchSize
3683
647
  });
648
+ for (const claim of recordingClaims) {
649
+ try {
650
+ await deleteObject(objectStorage, claim.objectKey);
651
+ const settled = await completeTranscriptionObject(db, claim);
652
+ if (!settled) {
653
+ throw new Error("cleanup claim no longer owns a transcription recording object");
654
+ }
655
+ deleted += 1;
656
+ } catch (error) {
657
+ failed += 1;
658
+ observability.warn(
659
+ "expired transcription recording object cleanup failed; claim remains reclaimable",
660
+ {
661
+ workspaceId: claim.workspaceId,
662
+ recordingId: claim.recordingId,
663
+ ...cleanupErrorAttributes(error)
664
+ }
665
+ );
666
+ }
667
+ }
668
+ let transcriptionRecordingsPurged = 0;
669
+ try {
670
+ transcriptionRecordingsPurged = await purgeTranscriptionRecordings(db, {
671
+ graceMs,
672
+ limit: batchSize
673
+ });
674
+ } catch (error) {
675
+ observability.warn("expired transcription recording metadata purge failed", {
676
+ ...cleanupErrorAttributes(error)
677
+ });
678
+ }
679
+ const claimed = claims.length + recordingClaims.length;
680
+ if (claimed > 0 || transcriptionRecordingsPurged > 0) {
681
+ observability.info("expired file upload cleanup swept", {
682
+ claimed,
683
+ deleted,
684
+ failed,
685
+ fileUploads: claims.length,
686
+ transcriptionRecordingObjects: recordingClaims.length,
687
+ transcriptionRecordingsPurged
688
+ });
689
+ }
690
+ return { claimed, deleted, failed };
3684
691
  }
3685
- if (summary.reconnectRequired) {
3686
- observability.incrementCounter({
3687
- name: "opengeni_knowledge_source_sync_reconnect_required_total",
3688
- help: "Knowledge-source synchronization failures requiring connector reconnection.",
3689
- labels: { provider }
692
+ async function recoverVideoGenerationWorkflows() {
693
+ const { db, startVideoGenerationWorkflow, observability } = await services();
694
+ if (!startVideoGenerationWorkflow) return { claimed: 0, started: 0, failed: 0 };
695
+ const claims = await claimVideoGenerationOperations(db, {
696
+ owner: `video-workflow-recovery:${randomUUID()}`,
697
+ leaseSeconds: 120,
698
+ limit: 100
3690
699
  });
3691
- }
3692
- if (failure) {
3693
- observability.incrementCounter({
3694
- name: "opengeni_knowledge_source_sync_failures_total",
3695
- help: "Knowledge-source synchronization terminal failures by bounded reason.",
3696
- labels: {
3697
- provider,
3698
- reason: failure.code,
3699
- retryable: failure.retryable ? "true" : "false"
700
+ let started = 0;
701
+ let failed = 0;
702
+ for (const claim of claims) {
703
+ try {
704
+ await startVideoGenerationWorkflow(claim);
705
+ started += 1;
706
+ } catch (error) {
707
+ failed += 1;
708
+ observability.warn("Video generation workflow recovery start failed", {
709
+ operationId: claim.operationId,
710
+ errorClass: error instanceof Error ? error.name : "UnknownError"
711
+ });
3700
712
  }
3701
- });
3702
- observability.warn("Knowledge source sync failed", {
3703
- provider,
3704
- outcome: failure.retryable ? "retryable_failed" : "failed",
3705
- reason: failure.code,
3706
- durationMs: summary.elapsedMs
3707
- });
3708
- return;
713
+ }
714
+ return { claimed: claims.length, started, failed };
3709
715
  }
3710
- observability.info("Knowledge source sync completed", {
3711
- provider,
3712
- outcome: "succeeded",
3713
- durationMs: summary.elapsedMs
3714
- });
716
+ return { reapExpiredFileUploads, recoverVideoGenerationWorkflows };
3715
717
  }
3716
- var SyncFailure = class extends Error {
3717
- constructor(code, retryable, reconnectRequired = false) {
3718
- super(code);
3719
- this.code = code;
3720
- this.retryable = retryable;
3721
- this.reconnectRequired = reconnectRequired;
3722
- this.name = "KnowledgeSourceSyncFailure";
3723
- }
3724
- };
3725
718
 
3726
719
  // src/activities/retained-screenshot-reaper.ts
3727
720
  import {
@@ -4037,7 +1030,7 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
4037
1030
  Math.max(1, startToCloseTimeoutMs - reserveMs)
4038
1031
  );
4039
1032
  }
4040
- const heartbeat2 = () => {
1033
+ const heartbeat = () => {
4041
1034
  if (!context) return;
4042
1035
  try {
4043
1036
  context.heartbeat({
@@ -4050,13 +1043,13 @@ function createRigVerificationActivityLifecycle(context = currentActivityContext
4050
1043
  if (!controller.signal.aborted) controller.abort(error);
4051
1044
  }
4052
1045
  };
4053
- heartbeat2();
1046
+ heartbeat();
4054
1047
  const heartbeatTimeoutMs = context?.info.heartbeatTimeoutMs;
4055
1048
  const heartbeatIntervalMs = typeof heartbeatTimeoutMs === "number" && heartbeatTimeoutMs > 0 ? Math.min(
4056
1049
  RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS,
4057
1050
  Math.max(RIG_VERIFICATION_HEARTBEAT_MIN_INTERVAL_MS, Math.floor(heartbeatTimeoutMs / 3))
4058
1051
  ) : RIG_VERIFICATION_HEARTBEAT_MAX_INTERVAL_MS;
4059
- const heartbeatTimer = context ? setInterval(heartbeat2, heartbeatIntervalMs) : null;
1052
+ const heartbeatTimer = context ? setInterval(heartbeat, heartbeatIntervalMs) : null;
4060
1053
  if (heartbeatTimer && "unref" in heartbeatTimer && typeof heartbeatTimer.unref === "function") {
4061
1054
  heartbeatTimer.unref();
4062
1055
  }
@@ -5188,7 +2181,7 @@ function createRigVerificationActivities(services) {
5188
2181
  }
5189
2182
 
5190
2183
  // src/activities/sandbox-lease.ts
5191
- import { createHash as createHash3, randomUUID as randomUUID2 } from "crypto";
2184
+ import { createHash, randomUUID as randomUUID2 } from "crypto";
5192
2185
  import { retainedProviderCommandPersistence } from "@opengeni/db/retained-provider-commands";
5193
2186
  import { Context } from "@temporalio/activity";
5194
2187
  import { OpLostReason, OpState } from "@opengeni/agent-proto";
@@ -5231,7 +2224,7 @@ import {
5231
2224
  retainedProcessSettlementIdentity,
5232
2225
  settleClaimedConnectedMachineBackgroundCommand,
5233
2226
  settleSandboxCheckpointArtifactGc,
5234
- rlsContextForWorkspace as rlsContextForWorkspace2,
2227
+ rlsContextForWorkspace,
5235
2228
  settleRetainedProcess
5236
2229
  } from "@opengeni/db";
5237
2230
  import {
@@ -5326,14 +2319,14 @@ function startSandboxReaperHeartbeat(details) {
5326
2319
  } catch {
5327
2320
  return () => void 0;
5328
2321
  }
5329
- const heartbeat2 = () => {
2322
+ const heartbeat = () => {
5330
2323
  try {
5331
2324
  context.heartbeat({ ...details, heartbeatAt: (/* @__PURE__ */ new Date()).toISOString() });
5332
2325
  } catch {
5333
2326
  }
5334
2327
  };
5335
- heartbeat2();
5336
- const timer = setInterval(heartbeat2, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
2328
+ heartbeat();
2329
+ const timer = setInterval(heartbeat, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
5337
2330
  if ("unref" in timer && typeof timer.unref === "function") timer.unref();
5338
2331
  return () => clearInterval(timer);
5339
2332
  }
@@ -5348,7 +2341,7 @@ function sandboxDrainCaptureId(operationId, attempt) {
5348
2341
  throw new Error("Sandbox drain activity attempt is invalid");
5349
2342
  }
5350
2343
  if (attempt === 1) return operationId.toLowerCase();
5351
- const bytes = createHash3("sha256").update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`).digest().subarray(0, 16);
2344
+ const bytes = createHash("sha256").update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`).digest().subarray(0, 16);
5352
2345
  bytes[6] = bytes[6] & 15 | 80;
5353
2346
  bytes[8] = bytes[8] & 63 | 128;
5354
2347
  const hex = bytes.toString("hex");
@@ -6745,7 +3738,7 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
6745
3738
  SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
6746
3739
  async (workspaceId) => {
6747
3740
  try {
6748
- const { accountId } = await rlsContextForWorkspace2(db, workspaceId);
3741
+ const { accountId } = await rlsContextForWorkspace(db, workspaceId);
6749
3742
  const balance = enforceBalance ? await getBillingBalance3(db, accountId) : { balanceMicros: 1 };
6750
3743
  const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
6751
3744
  workspaceId,
@@ -6883,7 +3876,7 @@ function sandboxDrainReason(lease) {
6883
3876
  return lease.rotationReason ?? "rotation_requested";
6884
3877
  }
6885
3878
  async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
6886
- const { accountId } = await rlsContextForWorkspace2(db, row.workspaceId);
3879
+ const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
6887
3880
  const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);
6888
3881
  if (!lease) {
6889
3882
  return false;
@@ -7491,12 +4484,11 @@ import {
7491
4484
  createSessionWithIdempotencyKeyResult,
7492
4485
  enqueueSessionWorkflowWakeIfRunnable,
7493
4486
  failScheduledGeneratedSessionRoute,
7494
- getScheduledTask as getScheduledTask2,
4487
+ getScheduledTask,
7495
4488
  getScheduledTaskCreatorPolicy,
7496
4489
  getScheduledTaskRunAcceptedExecution,
7497
4490
  getScheduledTaskRunByProducerKey,
7498
4491
  getScheduledTargetSessionExecution,
7499
- ensureKnowledgeSourceSyncState,
7500
4492
  getScheduledTaskPersonalConnectionDelegations,
7501
4493
  getScheduledTaskRunPersonalResourceAuthority,
7502
4494
  getScheduledTaskPersonalResourceAuthoritySubject,
@@ -7517,14 +4509,12 @@ import {
7517
4509
  listInstalledApiIntegrationServerIdsForDelegations,
7518
4510
  markScheduledTaskRunFailedIfQueued,
7519
4511
  markScheduledTaskRunSkippedIfQueued,
7520
- recordKnowledgeSourceSyncWake,
7521
- recordUsageEvent as recordUsageEvent3,
4512
+ recordUsageEvent as recordUsageEvent2,
7522
4513
  requireScheduledTaskIncidentAuthorityInTransaction,
7523
4514
  requireSession,
7524
4515
  requireWorkspace,
7525
4516
  settleScheduledTaskRunInTransaction,
7526
4517
  SessionSpawnDeniedDbError,
7527
- updateScheduledTaskRun as updateScheduledTaskRun2,
7528
4518
  updateSessionTitleWithEvent,
7529
4519
  upsertScheduledSessionGoalForRun,
7530
4520
  withSessionActivityRlsContext
@@ -7536,10 +4526,10 @@ import {
7536
4526
  resolveTurnExecutionPolicyV1 as resolveTurnExecutionPolicyV12
7537
4527
  } from "@opengeni/config";
7538
4528
  import { Context as Context2 } from "@temporalio/activity";
7539
- import { createHash as createHash5 } from "crypto";
4529
+ import { createHash as createHash3 } from "crypto";
7540
4530
 
7541
4531
  // src/scheduled-alert-occurrence.ts
7542
- import { createHash as createHash4 } from "crypto";
4532
+ import { createHash as createHash2 } from "crypto";
7543
4533
  var MAX_SCHEDULED_TASK_ID_BYTES = 256;
7544
4534
  var MAX_STARTS_AT_BYTES = 256;
7545
4535
  var MAX_PROVIDER_BYTES = 256;
@@ -7549,7 +4539,7 @@ var MAX_LABEL_KEY_BYTES = 256;
7549
4539
  var MAX_LABEL_VALUE_BYTES = 4096;
7550
4540
  var OFFSET_TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2})(?:\.(\d+))?)?(?:Z|[+-](\d{2}):(\d{2}))$/;
7551
4541
  function scheduledAlertResponderSessionCreateIdempotencyKey(input) {
7552
- const digest = createHash4("sha256").update(
4542
+ const digest = createHash2("sha256").update(
7553
4543
  JSON.stringify({
7554
4544
  version: 1,
7555
4545
  occurrenceKey: input.occurrence.sessionCreateIdempotencyKey,
@@ -7575,7 +4565,7 @@ function scheduledAlertOccurrenceIdentity(input) {
7575
4565
  MAX_PROVIDER_BYTES
7576
4566
  );
7577
4567
  if (!provider) return null;
7578
- const digest = createHash4("sha256").update(
4568
+ const digest = createHash2("sha256").update(
7579
4569
  JSON.stringify({
7580
4570
  version: 1,
7581
4571
  workspaceId: input.workspaceId,
@@ -7673,7 +4663,7 @@ function scheduledTaskRunProducerKey(input, suppliedActivityIdentity) {
7673
4663
  })}`;
7674
4664
  }
7675
4665
  function scheduledTaskGeneratedSessionCreateIdempotencyKey(producerKey) {
7676
- const digest = createHash5("sha256").update("scheduled-task-run\0", "utf8").update(producerKey, "utf8").digest("hex");
4666
+ const digest = createHash3("sha256").update("scheduled-task-run\0", "utf8").update(producerKey, "utf8").digest("hex");
7677
4667
  return `scheduled-task-run:${digest}`;
7678
4668
  }
7679
4669
  function scheduledTaskSessionTitle(taskName) {
@@ -7784,73 +4774,19 @@ function createScheduledTaskActivities(services) {
7784
4774
  return scheduledRunTerminalResult(priorRun.error);
7785
4775
  }
7786
4776
  }
7787
- const task = await getScheduledTask2(db, input.workspaceId, input.taskId);
4777
+ const task = await getScheduledTask(db, input.workspaceId, input.taskId);
7788
4778
  if (!task) {
7789
4779
  return { action: "deleted" };
7790
4780
  }
7791
- if (task.action.kind === "knowledge_source_sync") {
7792
- if (knowledgeSourceSyncEffectivelyPaused(task)) {
7793
- return { action: "blocked", reason: "knowledge_source_paused" };
7794
- }
7795
- const run = await createScheduledTaskRun(db, {
7796
- workspaceId: task.workspaceId,
7797
- taskId: task.id,
7798
- taskAuthorityRevision: task.authorityRevision,
7799
- taskExecutionDigest: task.executionDigest,
7800
- triggerType: input.triggerType,
7801
- producerKey: stableProducerKey,
7802
- scheduledAt: null
7803
- });
7804
- await ensureKnowledgeSourceSyncState(db, task);
7805
- await recordKnowledgeSourceSyncWake(db, {
7806
- accountId: task.accountId,
7807
- workspaceId: task.workspaceId,
7808
- sourceId: task.action.sourceId,
7809
- scheduledTaskId: task.id,
7810
- scheduledTaskRunId: run.id,
7811
- cause: input.triggerType,
7812
- producerKey: stableProducerKey,
7813
- sourceConfigGeneration: task.action.sourceConfigGeneration,
7814
- sourceLifecycleGeneration: task.action.sourceLifecycleGeneration
7815
- });
7816
- await recordUsageEvent3(db, {
7817
- accountId: task.accountId,
7818
- workspaceId: task.workspaceId,
7819
- eventType: "knowledge_source_sync.fired",
7820
- quantity: 1,
7821
- unit: "run",
7822
- sourceResourceType: "scheduled_task_run",
7823
- sourceResourceId: run.id,
7824
- initiator: input.initiator ?? {
7825
- kind: "service",
7826
- subjectId: "scheduler"
7827
- },
7828
- initiatorContext: {
7829
- scheduledTaskId: task.id,
7830
- scheduledTaskRunId: run.id
7831
- },
7832
- origin: "scheduled_task",
7833
- idempotencyKey: `usage:knowledge_source_sync.fired:${run.id}`
7834
- });
7835
- if (run.status === "queued") {
7836
- await updateScheduledTaskRun2(db, task.workspaceId, run.id, {
7837
- status: "dispatched",
7838
- actionKind: "knowledge_source_sync"
7839
- });
7840
- }
7841
- return {
7842
- action: "knowledge_source_sync",
7843
- accountId: task.accountId,
7844
- workspaceId: task.workspaceId,
7845
- taskId: task.id,
7846
- scheduledTaskRunId: run.id,
7847
- sourceId: task.action.sourceId,
7848
- overlapPolicy: task.overlapPolicy
7849
- };
4781
+ if (task.action.kind !== "agent_turn") {
4782
+ return { action: "blocked", reason: "legacy_source_schedule_requires_migration" };
7850
4783
  }
7851
4784
  if (task.status !== "active") {
7852
4785
  return { action: "blocked", reason: "scheduled_task_paused" };
7853
4786
  }
4787
+ if (task.agentConfig.knowledgeSource && knowledgeSourceSyncEffectivelyPaused(task)) {
4788
+ return { action: "blocked", reason: "knowledge_source_paused" };
4789
+ }
7854
4790
  const structuredAlertOccurrence = scheduledAlertOccurrenceIdentity({
7855
4791
  workspaceId: task.workspaceId,
7856
4792
  scheduledTaskId: task.id,
@@ -8112,6 +5048,11 @@ function createScheduledTaskActivities(services) {
8112
5048
  throw new Error("scheduled authority classes have different causal humans");
8113
5049
  }
8114
5050
  const causalHumanSubjectId = taskAuthoritySubjectId ?? xaiAuthoritySubjectId;
5051
+ if (task.agentConfig.knowledgeSource && causalHumanSubjectId !== task.agentConfig.knowledgeSource.initiatingSubjectId) {
5052
+ throw new Error(
5053
+ "Source schedule requires its connection owner's current human authorization; update the source schedule to authorize it"
5054
+ );
5055
+ }
8115
5056
  if ((acceptedVariableSet?.scope === "user" || acceptedRig?.scope === "user") && !causalHumanSubjectId) {
8116
5057
  throw new Error("scheduled personal-resource execution has no causal human");
8117
5058
  }
@@ -8256,7 +5197,7 @@ function createScheduledTaskActivities(services) {
8256
5197
  if (deferPublications) {
8257
5198
  return { kind: "replay", run, acceptedExecution };
8258
5199
  }
8259
- await recordUsageEvent3(dispatchDb, {
5200
+ await recordUsageEvent2(dispatchDb, {
8260
5201
  accountId: task.accountId,
8261
5202
  workspaceId: task.workspaceId,
8262
5203
  eventType: "agent_run.created",
@@ -8354,6 +5295,10 @@ function createScheduledTaskActivities(services) {
8354
5295
  firstPartyMcpTools,
8355
5296
  firstPartyMcpPermissions,
8356
5297
  ...creatorSessionPolicy,
5298
+ ...task.agentConfig.knowledgeSource?.destination.kind === "personal" ? {
5299
+ visibility: "user_private",
5300
+ scheduledSourceRunId: run.id
5301
+ } : {},
8357
5302
  metadata: {
8358
5303
  ...taskMetadata,
8359
5304
  model,
@@ -8842,7 +5787,7 @@ function createScheduledTaskActivities(services) {
8842
5787
  run: { ...run, status: "failed", error: terminalError.code }
8843
5788
  };
8844
5789
  }
8845
- await recordUsageEvent3(dispatchDb, {
5790
+ await recordUsageEvent2(dispatchDb, {
8846
5791
  accountId: task.accountId,
8847
5792
  workspaceId: task.workspaceId,
8848
5793
  eventType: "agent_run.created",
@@ -9062,7 +6007,7 @@ async function prepareIncidentTelemetrySource(input) {
9062
6007
  }
9063
6008
  async function recordScheduledTaskFiredUsage(db, acceptedExecution, run) {
9064
6009
  const task = acceptedExecution.task;
9065
- await recordUsageEvent3(db, {
6010
+ await recordUsageEvent2(db, {
9066
6011
  accountId: task.accountId,
9067
6012
  workspaceId: task.workspaceId,
9068
6013
  eventType: "scheduled_task.fired",
@@ -9209,6 +6154,10 @@ async function recoverBoundScheduledTaskDispatch(input) {
9209
6154
  firstPartyMcpTools: input.acceptedExecution.resolvedFirstPartyMcpTools,
9210
6155
  firstPartyMcpPermissions: input.acceptedExecution.resolvedFirstPartyMcpPermissions,
9211
6156
  ...scheduledCreatorSessionPolicyInput(recoveredCreatorPolicy?.sessionPolicy ?? null),
6157
+ ...task.agentConfig.knowledgeSource?.destination.kind === "personal" ? {
6158
+ visibility: "user_private",
6159
+ scheduledSourceRunId: input.run.id
6160
+ } : {},
9212
6161
  metadata: {
9213
6162
  ...taskMetadata,
9214
6163
  model: input.acceptedExecution.resolvedModel,
@@ -9486,7 +6435,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
9486
6435
  scheduledUpdate.events
9487
6436
  );
9488
6437
  }
9489
- await recordUsageEvent3(input.db, {
6438
+ await recordUsageEvent2(input.db, {
9490
6439
  accountId: task.accountId,
9491
6440
  workspaceId: task.workspaceId,
9492
6441
  eventType: "agent_run.created",
@@ -9534,7 +6483,7 @@ async function replayScheduledTaskDispatch(input) {
9534
6483
  const sessionId = input.run.sessionId;
9535
6484
  const triggerEventId = input.run.triggerEventId;
9536
6485
  await requireSession(input.db, input.task.workspaceId, sessionId);
9537
- await recordUsageEvent3(input.db, {
6486
+ await recordUsageEvent2(input.db, {
9538
6487
  accountId: input.task.accountId,
9539
6488
  workspaceId: input.task.workspaceId,
9540
6489
  eventType: "agent_run.created",
@@ -9904,14 +6853,14 @@ function createSessionStateActivities(services, overrides = {}) {
9904
6853
  return { action: "failed" };
9905
6854
  }
9906
6855
  const requestedRetryDelayMs = input.retryDelayMs;
9907
- const retryDelayMs2 = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
6856
+ const retryDelayMs = typeof requestedRetryDelayMs === "number" && Number.isFinite(requestedRetryDelayMs) ? Math.max(1e3, Math.min(6e4, Math.trunc(requestedRetryDelayMs))) : 6e4;
9908
6857
  await enqueueSessionWorkflowWakeFn(db, {
9909
6858
  accountId: input.accountId,
9910
6859
  workspaceId: input.workspaceId,
9911
6860
  sessionId: input.sessionId,
9912
6861
  temporalWorkflowId: workflowId,
9913
6862
  reason: "turn_activity_failed_before_attempt_claim",
9914
- notBefore: new Date(Date.now() + retryDelayMs2)
6863
+ notBefore: new Date(Date.now() + retryDelayMs)
9915
6864
  });
9916
6865
  return { action: "unclaimed" };
9917
6866
  }
@@ -10340,6 +7289,7 @@ function createControlActivitiesFromServices(services, resolveDocumentServices)
10340
7289
  return {
10341
7290
  ...createAutomationActivities(services),
10342
7291
  ...createDocumentActivities(services, resolveDocumentServices),
7292
+ ...createKnowledgeIndexingActivities(services, resolveDocumentServices),
10343
7293
  ...createKnowledgeSourceSyncActivities(services, resolveDocumentServices),
10344
7294
  ...createSessionStateActivities(services),
10345
7295
  ...createScheduledTaskActivities(services),
@@ -10369,4 +7319,4 @@ export {
10369
7319
  createControlActivities,
10370
7320
  createControlActivitiesFromServices
10371
7321
  };
10372
- //# sourceMappingURL=activities-control-MKGEOBXT.js.map
7322
+ //# sourceMappingURL=activities-control-HMFGSKC5.js.map