@opengeni/worker-bundle 0.20.1 → 0.22.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +52 -0
- package/dist/activities/agent-turn/agent-build.d.ts +89 -0
- package/dist/activities/agent-turn/claim.d.ts +67 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +40 -0
- package/dist/activities/agent-turn/codex.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +88 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +56 -0
- package/dist/activities/agent-turn/errors.d.ts +253 -0
- package/dist/activities/agent-turn/failure-settlement.d.ts +33 -0
- package/dist/activities/agent-turn/file-resources.d.ts +37 -0
- package/dist/activities/agent-turn/finalization.d.ts +42 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +105 -0
- package/dist/activities/agent-turn/governance-model.d.ts +66 -0
- package/dist/activities/agent-turn/history-sink.d.ts +52 -0
- package/dist/activities/agent-turn/history.d.ts +63 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +145 -0
- package/dist/activities/agent-turn/model-usage.d.ts +218 -0
- package/dist/activities/agent-turn/quiescence.d.ts +135 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +84 -0
- package/dist/activities/agent-turn/run.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-establish.d.ts +93 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +89 -0
- package/dist/activities/agent-turn/sandbox-route.d.ts +151 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +44 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +107 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +142 -0
- package/dist/activities/agent-turn/tool-policy.d.ts +127 -0
- package/dist/activities/agent-turn/tool-result-spill.d.ts +59 -0
- package/dist/activities/agent-turn/turn-context.d.ts +144 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +2 -0
- package/dist/activities/agent-turn.d.ts +12 -954
- package/dist/activities/automations.d.ts +16 -0
- package/dist/activities/common.d.ts +1 -1
- package/dist/activities/connector-attachments.d.ts +13 -0
- package/dist/activities/documents.d.ts +1 -0
- package/dist/activities/environment.d.ts +26 -15
- package/dist/activities/generated-images.d.ts +1 -1
- package/dist/activities/git-credential-renewal.d.ts +3 -1
- package/dist/activities/goals.d.ts +4 -1
- package/dist/activities/google-drive-provider.d.ts +29 -0
- package/dist/activities/google-drive-publication.d.ts +66 -0
- package/dist/activities/image-generation-references.d.ts +2 -0
- package/dist/activities/incident-telemetry-authority.d.ts +52 -0
- package/dist/activities/incident-telemetry-preflight.d.ts +71 -0
- package/dist/activities/knowledge-source-sync-driver.d.ts +21 -0
- package/dist/activities/knowledge-source-sync.d.ts +33 -1
- package/dist/activities/mcp-credentials.d.ts +9 -3
- package/dist/activities/open-suffix-resume.d.ts +77 -0
- package/dist/activities/packs.d.ts +3 -11
- package/dist/activities/parent-wake.d.ts +19 -0
- package/dist/activities/retained-screenshot-reaper.d.ts +1 -1
- package/dist/activities/retained-screenshots.d.ts +1 -1
- package/dist/activities/rig-verification.d.ts +1 -1
- package/dist/activities/run-input.d.ts +16 -7
- package/dist/activities/sandbox-images.d.ts +1 -14
- package/dist/activities/sandbox-lease.d.ts +21 -3
- package/dist/activities/scheduled-tasks.d.ts +47 -0
- package/dist/activities/types.d.ts +23 -2
- package/dist/activities/video-media-validation.d.ts +1 -1
- package/dist/activities/video-output-retention.d.ts +1 -2
- package/dist/activities/xai-auth.d.ts +3 -0
- package/dist/{activities-control-SMKF2OBE.js → activities-control-DKG22ONJ.js} +2946 -549
- package/dist/activities-control-DKG22ONJ.js.map +1 -0
- package/dist/activities-control.d.ts +6 -2
- package/dist/activities-turn-4IMC3MAA.js +21326 -0
- package/dist/activities-turn-4IMC3MAA.js.map +1 -0
- package/dist/activities.d.ts +7 -3
- package/dist/artifact-materializer-entry.js +41 -22
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/{chunk-N5TJIFCE.js → chunk-CE7TPKBS.js} +468 -1
- package/dist/chunk-CE7TPKBS.js.map +1 -0
- package/dist/chunk-CNYGVRKF.js +4698 -0
- package/dist/chunk-CNYGVRKF.js.map +1 -0
- package/dist/connection-provider-use.d.ts +18 -0
- package/dist/editable-artifact-materializer.d.ts +4 -3
- package/dist/github-rest-mcp.d.ts +26 -0
- package/dist/index.d.ts +10 -2
- package/dist/index.js +58 -14
- package/dist/index.js.map +1 -1
- package/dist/knowledge-source-sync-activity-policy.d.ts +2 -0
- package/dist/model-context-contributions.d.ts +8 -4
- package/dist/observability-metrics.d.ts +63 -4
- package/dist/opensandbox-kubernetes-inventory.d.ts +21 -0
- package/dist/personal-github-git-credentials.d.ts +9 -0
- package/dist/pr-review-credentials.d.ts +6 -0
- package/dist/sandbox-archive-storage.d.ts +45 -0
- package/dist/sandbox-resume.d.ts +65 -6
- package/dist/sandbox-routing.d.ts +38 -2
- package/dist/scheduled-alert-occurrence.d.ts +33 -0
- package/dist/workflow-bundle.js +87 -28
- package/dist/workflows/activities.d.ts +1 -0
- package/dist/workflows/automations.d.ts +6 -0
- package/dist/workflows/document-index.d.ts +1 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +292 -0
- package/src/activities/agent-turn/agent-build.ts +802 -0
- package/src/activities/agent-turn/claim.ts +590 -0
- package/src/activities/agent-turn/codex-capacity.ts +720 -0
- package/src/activities/agent-turn/codex.ts +94 -0
- package/src/activities/agent-turn/compaction-prep.ts +722 -0
- package/src/activities/agent-turn/credential-leases.ts +240 -0
- package/src/activities/agent-turn/errors.ts +1068 -0
- package/src/activities/agent-turn/failure-settlement.ts +1436 -0
- package/src/activities/agent-turn/file-resources.ts +227 -0
- package/src/activities/agent-turn/finalization.ts +754 -0
- package/src/activities/agent-turn/github-repository-bindings.ts +300 -0
- package/src/activities/agent-turn/governance-model.ts +465 -0
- package/src/activities/agent-turn/history-sink.ts +139 -0
- package/src/activities/agent-turn/history.ts +200 -0
- package/src/activities/agent-turn/media-artifacts.ts +359 -0
- package/src/activities/agent-turn/model-usage.ts +823 -0
- package/src/activities/agent-turn/quiescence.ts +353 -0
- package/src/activities/agent-turn/run-credentials.ts +761 -0
- package/src/activities/agent-turn/run.ts +1530 -0
- package/src/activities/agent-turn/sandbox-establish.ts +1207 -0
- package/src/activities/agent-turn/sandbox-provision.ts +569 -0
- package/src/activities/agent-turn/sandbox-route.ts +358 -0
- package/src/activities/agent-turn/sandbox-runtime.ts +731 -0
- package/src/activities/agent-turn/stream-attempt.ts +1670 -0
- package/src/activities/agent-turn/tool-environment.ts +729 -0
- package/src/activities/agent-turn/tool-policy.ts +216 -0
- package/src/activities/agent-turn/tool-result-spill.ts +265 -0
- package/src/activities/agent-turn/turn-context.ts +281 -0
- package/src/activities/agent-turn/xai-capacity.ts +229 -0
- package/src/activities/agent-turn.ts +12 -12575
- package/src/activities/api-integrations.ts +3 -0
- package/src/activities/automations.ts +214 -0
- package/src/activities/codemode-dispatcher.ts +12 -1
- package/src/activities/common.ts +1 -5
- package/src/activities/connector-attachments.ts +168 -0
- package/src/activities/context-compaction.ts +18 -0
- package/src/activities/documents.ts +6 -4
- package/src/activities/environment.ts +151 -55
- package/src/activities/generated-images.ts +26 -11
- package/src/activities/git-credential-renewal.ts +13 -3
- package/src/activities/goals.ts +85 -19
- package/src/activities/google-drive-provider.ts +178 -0
- package/src/activities/google-drive-publication.ts +938 -0
- package/src/activities/image-generation-references.ts +11 -2
- package/src/activities/incident-telemetry-authority.ts +327 -0
- package/src/activities/incident-telemetry-preflight.ts +317 -0
- package/src/activities/knowledge-source-sync-driver.ts +23 -0
- package/src/activities/knowledge-source-sync.ts +556 -40
- package/src/activities/mcp-credentials.ts +191 -17
- package/src/activities/open-suffix-resume.ts +422 -0
- package/src/activities/packs.ts +12 -60
- package/src/activities/parent-wake.ts +64 -14
- package/src/activities/retained-screenshot-reaper.ts +6 -3
- package/src/activities/retained-screenshots.ts +15 -47
- package/src/activities/rig-verification.ts +1 -1
- package/src/activities/run-input.ts +128 -136
- package/src/activities/sandbox-images.ts +7 -166
- package/src/activities/sandbox-lease.ts +406 -15
- package/src/activities/scheduled-tasks.ts +2103 -411
- package/src/activities/session-state.ts +10 -2
- package/src/activities/types.ts +32 -2
- package/src/activities/video-media-validation.ts +3 -3
- package/src/activities/video-output-retention.ts +2 -57
- package/src/activities/xai-auth.ts +8 -0
- package/src/activities-control.ts +2 -0
- package/src/activities.ts +3 -0
- package/src/activity-services.ts +13 -4
- package/src/connection-provider-use.ts +36 -0
- package/src/editable-artifact-materializer-subprocess.ts +25 -14
- package/src/editable-artifact-materializer.ts +29 -18
- package/src/github-rest-mcp.ts +713 -0
- package/src/http.ts +16 -6
- package/src/index.ts +40 -5
- package/src/knowledge-source-sync-activity-policy.ts +2 -0
- package/src/model-context-contributions.ts +33 -10
- package/src/observability-metrics.ts +357 -0
- package/src/opensandbox-kubernetes-inventory.ts +292 -0
- package/src/personal-github-git-credentials.ts +331 -0
- package/src/pr-review-credentials.ts +163 -0
- package/src/sandbox-archive-storage.ts +159 -0
- package/src/sandbox-resume.ts +449 -63
- package/src/sandbox-routing.ts +314 -16
- package/src/scheduled-alert-occurrence.ts +176 -0
- package/src/workflows/activities.ts +18 -2
- package/src/workflows/automations.ts +11 -0
- package/src/workflows/session.ts +13 -3
- package/src/workflows.ts +1 -0
- package/dist/activities-control-SMKF2OBE.js.map +0 -1
- package/dist/activities-turn-IDS3WEHA.js +0 -16532
- package/dist/activities-turn-IDS3WEHA.js.map +0 -1
- package/dist/chunk-N5TJIFCE.js.map +0 -1
- package/dist/chunk-P6Z2SLAA.js +0 -3092
- package/dist/chunk-P6Z2SLAA.js.map +0 -1
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { heartbeat } from "@temporalio/activity";
|
|
3
|
-
import
|
|
3
|
+
import {
|
|
4
|
+
googleDriveProviderRetryOptions,
|
|
5
|
+
type GoogleDriveProviderRetryOptions,
|
|
6
|
+
type Settings,
|
|
7
|
+
} from "@opengeni/config";
|
|
4
8
|
import {
|
|
5
9
|
KnowledgeSourceSyncRunSummary,
|
|
6
10
|
type ScheduledTask,
|
|
@@ -39,6 +43,7 @@ import {
|
|
|
39
43
|
appendKnowledgeDocumentVersion,
|
|
40
44
|
appendKnowledgeSourceAclVersion,
|
|
41
45
|
beginKnowledgeSyncRun,
|
|
46
|
+
beginGoogleDriveObjectAclRefresh,
|
|
42
47
|
buildConnectionTokenResolver,
|
|
43
48
|
checkpointKnowledgeSourceSync,
|
|
44
49
|
claimKnowledgeSourceSyncLease,
|
|
@@ -60,6 +65,7 @@ import {
|
|
|
60
65
|
KnowledgeSourceSyncObservationFenceError,
|
|
61
66
|
reconcileKnowledgeSourceSyncCompleteScan,
|
|
62
67
|
reconcileKnowledgeSourceSyncLiveGeneration,
|
|
68
|
+
recordGoogleDriveObjectAclEvidence,
|
|
63
69
|
recordKnowledgeSourceSyncItemOutcomes,
|
|
64
70
|
recordKnowledgeSourceSyncObjectObservations,
|
|
65
71
|
recordUsageEvent,
|
|
@@ -77,13 +83,22 @@ import {
|
|
|
77
83
|
type KnowledgeSourceSyncObservationFloor,
|
|
78
84
|
} from "@opengeni/db";
|
|
79
85
|
import { readResponseJsonBounded } from "@opengeni/network";
|
|
86
|
+
import type { Observability } from "@opengeni/observability";
|
|
80
87
|
import { createDocumentActivities } from "./documents";
|
|
88
|
+
import {
|
|
89
|
+
fetchGoogleDriveProvider,
|
|
90
|
+
GoogleDriveProviderTransportError,
|
|
91
|
+
type GoogleDriveProviderOperation,
|
|
92
|
+
} from "./google-drive-provider";
|
|
81
93
|
import type {
|
|
82
94
|
ControlActivityServices,
|
|
83
95
|
RunKnowledgeSourceSyncBatchInput,
|
|
84
96
|
RunKnowledgeSourceSyncBatchResult,
|
|
85
97
|
} from "./types";
|
|
86
|
-
import type {
|
|
98
|
+
import type {
|
|
99
|
+
KnowledgeSourceSyncAclEvidence,
|
|
100
|
+
KnowledgeSourceSyncDriver,
|
|
101
|
+
} from "./knowledge-source-sync-driver";
|
|
87
102
|
import {
|
|
88
103
|
advanceGoogleDriveChangesCursor,
|
|
89
104
|
buildGoogleDriveChangesCursor,
|
|
@@ -104,12 +119,31 @@ const DRIVE_JSON_MAX_BYTES = 2 * 1024 * 1024;
|
|
|
104
119
|
const GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS = 24 * 60 * 60 * 1_000;
|
|
105
120
|
const GOOGLE_DRIVE_SYNC_INVOCATION_SLICE_MS = 30_000;
|
|
106
121
|
const GOOGLE_DRIVE_EXECUTION_CHECKPOINT_MAX_BYTES = 2 * 1024 * 1024;
|
|
122
|
+
const GOOGLE_DRIVE_ACL_FRESHNESS_MS = 26 * 60 * 60 * 1_000;
|
|
123
|
+
const GOOGLE_DRIVE_ACL_MAX_PRINCIPALS = 1_000;
|
|
107
124
|
const ATLASSIAN_API_BASE = "https://api.atlassian.com";
|
|
108
125
|
const ATLASSIAN_JSON_MAX_BYTES = 4 * 1024 * 1024;
|
|
109
126
|
|
|
110
127
|
type KnowledgeSyncEntry = GoogleDriveInventoryEntry | AtlassianInventoryEntry;
|
|
111
128
|
type KnowledgeSyncStopReason = GoogleDriveInventoryStopReason | AtlassianInventoryStopReason;
|
|
112
129
|
|
|
130
|
+
type GoogleDrivePermission = {
|
|
131
|
+
id: string;
|
|
132
|
+
type: "user" | "group" | "domain" | "anyone";
|
|
133
|
+
role: "owner" | "organizer" | "fileOrganizer" | "writer" | "commenter" | "reader";
|
|
134
|
+
emailAddress: string | null;
|
|
135
|
+
domain: string | null;
|
|
136
|
+
allowFileDiscovery: boolean | null;
|
|
137
|
+
expirationTime: string | null;
|
|
138
|
+
inherited: boolean;
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
type GoogleDrivePermissionsPage = {
|
|
142
|
+
permissions: GoogleDrivePermission[];
|
|
143
|
+
nextPageToken: string | null;
|
|
144
|
+
denied: boolean;
|
|
145
|
+
};
|
|
146
|
+
|
|
113
147
|
export function knowledgeSyncObservationPolicy(kind: "google_drive" | "atlassian") {
|
|
114
148
|
return kind === "google_drive"
|
|
115
149
|
? { revisionOrdering: "canonical_decimal" as const, filterWithDriveDurability: true }
|
|
@@ -188,6 +222,10 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
188
222
|
reconnectRequired: false,
|
|
189
223
|
failures: [],
|
|
190
224
|
};
|
|
225
|
+
observability.info("Knowledge source sync started", {
|
|
226
|
+
provider: providerLabel,
|
|
227
|
+
outcome: "started",
|
|
228
|
+
});
|
|
191
229
|
try {
|
|
192
230
|
summary.phase = "inventory";
|
|
193
231
|
const resolved = await getKnowledgeSourceForSyncAuthority(db, {
|
|
@@ -279,6 +317,7 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
279
317
|
const provider = await resolveKnowledgeSyncProvider({
|
|
280
318
|
db,
|
|
281
319
|
settings,
|
|
320
|
+
observability,
|
|
282
321
|
accountId: input.accountId,
|
|
283
322
|
workspaceId: input.workspaceId,
|
|
284
323
|
action,
|
|
@@ -437,7 +476,26 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
437
476
|
if (obligation.documentId !== details.version.documentId) {
|
|
438
477
|
throw new SyncFailure("authority_changed", false);
|
|
439
478
|
}
|
|
440
|
-
if (obligation.status === "indexed") return obligation;
|
|
479
|
+
if (obligation.status === "indexed" && !details.indexRequired) return obligation;
|
|
480
|
+
if (obligation.status === "indexed" && details.indexRequired) {
|
|
481
|
+
summary.phase = "index";
|
|
482
|
+
try {
|
|
483
|
+
const indexedDocument = await documentActivities.indexDocument({
|
|
484
|
+
accountId: input.accountId,
|
|
485
|
+
workspaceId: input.workspaceId,
|
|
486
|
+
documentId: details.version.documentId,
|
|
487
|
+
authorityKind: action.destination.kind,
|
|
488
|
+
authorityWorkspaceId: action.destination.workspaceId,
|
|
489
|
+
authoritySubjectId: action.destination.subjectId,
|
|
490
|
+
});
|
|
491
|
+
if (indexedDocument.status !== "ready") {
|
|
492
|
+
throw new Error("document indexing did not complete");
|
|
493
|
+
}
|
|
494
|
+
} catch {
|
|
495
|
+
throw new SyncFailure("indexing_failed", true);
|
|
496
|
+
}
|
|
497
|
+
return obligation;
|
|
498
|
+
}
|
|
441
499
|
if (obligation.status === "failed") {
|
|
442
500
|
const retried = await retryKnowledgeSourceSyncIndexObligation(db, {
|
|
443
501
|
accountId: input.accountId,
|
|
@@ -491,6 +549,74 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
491
549
|
return { ...obligation, status: "indexed" };
|
|
492
550
|
};
|
|
493
551
|
|
|
552
|
+
const recordEntryAcl = async (details: {
|
|
553
|
+
entry: KnowledgeSyncEntry;
|
|
554
|
+
object: { id: string; lifecycleGeneration: number };
|
|
555
|
+
version: { id: string; versionGeneration: number };
|
|
556
|
+
obligation: { id: string; sourceSyncGeneration: number };
|
|
557
|
+
}): Promise<{
|
|
558
|
+
aclEligibility: "pending" | "eligible" | "denied";
|
|
559
|
+
aclEvidence: Record<string, unknown> | null;
|
|
560
|
+
}> => {
|
|
561
|
+
if (!driver.readAcl || !provider.aclAuthority) {
|
|
562
|
+
return { aclEligibility: "pending", aclEvidence: null };
|
|
563
|
+
}
|
|
564
|
+
await beginGoogleDriveObjectAclRefresh(db, {
|
|
565
|
+
accountId: input.accountId,
|
|
566
|
+
workspaceId: input.workspaceId,
|
|
567
|
+
obligationId: details.obligation.id,
|
|
568
|
+
sourceSyncGeneration: details.obligation.sourceSyncGeneration,
|
|
569
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
570
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
571
|
+
objectLifecycleGeneration: details.object.lifecycleGeneration,
|
|
572
|
+
objectVersionGeneration: details.version.versionGeneration,
|
|
573
|
+
});
|
|
574
|
+
const remainingProviderRequests =
|
|
575
|
+
action.limits.maxProviderRequests - summary.providerRequests;
|
|
576
|
+
if (remainingProviderRequests < 1) {
|
|
577
|
+
throw new SyncFailure("resource_limit", false);
|
|
578
|
+
}
|
|
579
|
+
const evidence = await driver.readAcl(details.entry, remainingProviderRequests);
|
|
580
|
+
summary.providerRequests += evidence.providerRequests;
|
|
581
|
+
if (summary.providerRequests > action.limits.maxProviderRequests) {
|
|
582
|
+
throw new SyncFailure("resource_limit", false);
|
|
583
|
+
}
|
|
584
|
+
const recorded = await recordGoogleDriveObjectAclEvidence(db, {
|
|
585
|
+
accountId: input.accountId,
|
|
586
|
+
workspaceId: input.workspaceId,
|
|
587
|
+
obligationId: details.obligation.id,
|
|
588
|
+
connectionId: provider.aclAuthority.connectionId,
|
|
589
|
+
connectionVersion: provider.aclAuthority.connectionVersion,
|
|
590
|
+
sourceGooglePermissionId: provider.aclAuthority.sourceGooglePermissionId,
|
|
591
|
+
sourceSyncGeneration: details.obligation.sourceSyncGeneration,
|
|
592
|
+
sourceConfigGeneration: action.sourceConfigGeneration,
|
|
593
|
+
sourceLifecycleGeneration: action.sourceLifecycleGeneration,
|
|
594
|
+
objectLifecycleGeneration: details.object.lifecycleGeneration,
|
|
595
|
+
objectVersionGeneration: details.version.versionGeneration,
|
|
596
|
+
providerRevision: evidence.providerRevision,
|
|
597
|
+
driveId: evidence.driveId,
|
|
598
|
+
aclRevision: evidence.aclRevision,
|
|
599
|
+
eligibility: evidence.eligibility,
|
|
600
|
+
observedAt: evidence.observedAt,
|
|
601
|
+
expiresAt: evidence.expiresAt,
|
|
602
|
+
citationLocator: driver.citationLocator(details.entry),
|
|
603
|
+
operationId: `drive-acl:${details.obligation.id}:${evidence.aclRevision}`,
|
|
604
|
+
principals: evidence.principals,
|
|
605
|
+
});
|
|
606
|
+
return {
|
|
607
|
+
aclEligibility: evidence.eligibility,
|
|
608
|
+
aclEvidence: {
|
|
609
|
+
version: 1,
|
|
610
|
+
provider: "google_drive",
|
|
611
|
+
evidenceId: recorded.evidenceId,
|
|
612
|
+
aclRevision: evidence.aclRevision,
|
|
613
|
+
aclHash: recorded.aclHash,
|
|
614
|
+
observedAt: evidence.observedAt,
|
|
615
|
+
expiresAt: evidence.expiresAt,
|
|
616
|
+
},
|
|
617
|
+
};
|
|
618
|
+
};
|
|
619
|
+
|
|
494
620
|
const outcomes = [];
|
|
495
621
|
for (const entry of entriesToProcess) {
|
|
496
622
|
heartbeat({ sourceId: input.sourceId, externalObjectId: entry.externalObjectId });
|
|
@@ -609,9 +735,15 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
609
735
|
entry,
|
|
610
736
|
object,
|
|
611
737
|
version: currentVersion,
|
|
612
|
-
indexRequired:
|
|
738
|
+
indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0,
|
|
613
739
|
obligation: currentObligation,
|
|
614
740
|
});
|
|
741
|
+
const objectAcl = await recordEntryAcl({
|
|
742
|
+
entry,
|
|
743
|
+
object,
|
|
744
|
+
version: currentVersion,
|
|
745
|
+
obligation,
|
|
746
|
+
});
|
|
615
747
|
summary.unchanged += 1;
|
|
616
748
|
outcomes.push({
|
|
617
749
|
externalObjectId: entry.externalObjectId,
|
|
@@ -619,7 +751,8 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
619
751
|
contentSha256: currentVersion.contentSha256,
|
|
620
752
|
providerRevision: entry.externalVersionId,
|
|
621
753
|
metadataHash,
|
|
622
|
-
aclEligibility:
|
|
754
|
+
aclEligibility: objectAcl.aclEligibility,
|
|
755
|
+
aclEvidence: objectAcl.aclEvidence,
|
|
623
756
|
indexObligationId: obligation.id,
|
|
624
757
|
});
|
|
625
758
|
continue;
|
|
@@ -706,9 +839,10 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
706
839
|
version,
|
|
707
840
|
indexRequired: (currentObservation?.documentChunkCount ?? 0) === 0,
|
|
708
841
|
});
|
|
842
|
+
const objectAcl = await recordEntryAcl({ entry, object, version, obligation });
|
|
709
843
|
summary.unchanged += 1;
|
|
710
844
|
summary.indexed += 1;
|
|
711
|
-
summary.aclPending += 1;
|
|
845
|
+
if (objectAcl.aclEligibility === "pending") summary.aclPending += 1;
|
|
712
846
|
outcomes.push({
|
|
713
847
|
externalObjectId: entry.externalObjectId,
|
|
714
848
|
outcome: "unchanged" as const,
|
|
@@ -716,7 +850,8 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
716
850
|
sizeBytes: bytes.byteLength,
|
|
717
851
|
providerRevision: entry.externalVersionId,
|
|
718
852
|
metadataHash,
|
|
719
|
-
aclEligibility:
|
|
853
|
+
aclEligibility: objectAcl.aclEligibility,
|
|
854
|
+
aclEvidence: objectAcl.aclEvidence,
|
|
720
855
|
indexObligationId: obligation.id,
|
|
721
856
|
});
|
|
722
857
|
continue;
|
|
@@ -806,9 +941,10 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
806
941
|
version,
|
|
807
942
|
indexRequired: true,
|
|
808
943
|
});
|
|
944
|
+
const objectAcl = await recordEntryAcl({ entry, object, version, obligation });
|
|
809
945
|
summary.indexed += 1;
|
|
810
946
|
summary.imported += 1;
|
|
811
|
-
summary.aclPending += 1;
|
|
947
|
+
if (objectAcl.aclEligibility === "pending") summary.aclPending += 1;
|
|
812
948
|
summary.bytes += bytes.byteLength;
|
|
813
949
|
outcomes.push({
|
|
814
950
|
externalObjectId: entry.externalObjectId,
|
|
@@ -817,7 +953,8 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
817
953
|
sizeBytes: bytes.byteLength,
|
|
818
954
|
providerRevision: entry.externalVersionId,
|
|
819
955
|
metadataHash,
|
|
820
|
-
aclEligibility:
|
|
956
|
+
aclEligibility: objectAcl.aclEligibility,
|
|
957
|
+
aclEvidence: objectAcl.aclEvidence,
|
|
821
958
|
indexObligationId: obligation.id,
|
|
822
959
|
});
|
|
823
960
|
} catch (error) {
|
|
@@ -990,6 +1127,7 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
990
1127
|
labels: { provider: providerLabel },
|
|
991
1128
|
amount: summary.bytes,
|
|
992
1129
|
});
|
|
1130
|
+
recordSyncHealthTelemetry(observability, providerLabel, summary, "succeeded");
|
|
993
1131
|
return {
|
|
994
1132
|
action: "complete",
|
|
995
1133
|
bufferedWake: settled.bufferedWake,
|
|
@@ -1077,6 +1215,7 @@ export function createKnowledgeSourceSyncActivities(
|
|
|
1077
1215
|
labels: { provider: providerLabel },
|
|
1078
1216
|
amount: summary.bytes,
|
|
1079
1217
|
});
|
|
1218
|
+
recordSyncHealthTelemetry(observability, providerLabel, summary, "failed", failure);
|
|
1080
1219
|
return {
|
|
1081
1220
|
action: "failed",
|
|
1082
1221
|
bufferedWake: settled.bufferedWake,
|
|
@@ -1201,11 +1340,17 @@ export type GoogleDriveSyncProviderPort = {
|
|
|
1201
1340
|
pageToken: string | null;
|
|
1202
1341
|
pageSize: number;
|
|
1203
1342
|
}) => Promise<GoogleDriveInventoryPage>;
|
|
1343
|
+
listPermissions?: (input: {
|
|
1344
|
+
fileId: string;
|
|
1345
|
+
pageToken: string | null;
|
|
1346
|
+
pageSize: number;
|
|
1347
|
+
}) => Promise<GoogleDrivePermissionsPage>;
|
|
1204
1348
|
};
|
|
1205
1349
|
|
|
1206
1350
|
async function resolveKnowledgeSyncProvider(input: {
|
|
1207
1351
|
db: Database;
|
|
1208
1352
|
settings: Settings;
|
|
1353
|
+
observability: Observability;
|
|
1209
1354
|
accountId: string;
|
|
1210
1355
|
workspaceId: string;
|
|
1211
1356
|
action: Extract<ScheduledTask["action"], { kind: "knowledge_source_sync" }>;
|
|
@@ -1217,6 +1362,11 @@ async function resolveKnowledgeSyncProvider(input: {
|
|
|
1217
1362
|
GoogleDriveSelectedSource["destination"] | AtlassianSelectedSource["destination"]
|
|
1218
1363
|
>;
|
|
1219
1364
|
driver: KnowledgeSourceSyncDriver<KnowledgeSyncEntry, KnowledgeSyncStopReason>;
|
|
1365
|
+
aclAuthority: {
|
|
1366
|
+
connectionId: string;
|
|
1367
|
+
connectionVersion: number;
|
|
1368
|
+
sourceGooglePermissionId: string;
|
|
1369
|
+
} | null;
|
|
1220
1370
|
}> {
|
|
1221
1371
|
const { action } = input;
|
|
1222
1372
|
const connection = await getConnectionMetadata(
|
|
@@ -1341,6 +1491,11 @@ async function resolveKnowledgeSyncProvider(input: {
|
|
|
1341
1491
|
return {
|
|
1342
1492
|
kind: "google_drive",
|
|
1343
1493
|
selectedDestination: source.destination!,
|
|
1494
|
+
aclAuthority: {
|
|
1495
|
+
connectionId: resolvedConnection.id,
|
|
1496
|
+
connectionVersion: resolvedConnection.version,
|
|
1497
|
+
sourceGooglePermissionId: metadata.googlePermissionId,
|
|
1498
|
+
},
|
|
1344
1499
|
driver: googleDriveSyncDriver({
|
|
1345
1500
|
actionProviderCoordinationKey: action.providerCoordinationKey,
|
|
1346
1501
|
metadata,
|
|
@@ -1349,6 +1504,8 @@ async function resolveKnowledgeSyncProvider(input: {
|
|
|
1349
1504
|
workspaceId: input.workspaceId,
|
|
1350
1505
|
initiatingSubjectId: action.initiatingSubjectId,
|
|
1351
1506
|
authorization,
|
|
1507
|
+
retry: googleDriveProviderRetryOptions(input.settings),
|
|
1508
|
+
observability: input.observability,
|
|
1352
1509
|
limits: action.limits,
|
|
1353
1510
|
connectionId: resolvedConnection.id,
|
|
1354
1511
|
fullReconciliationIntervalMs: GOOGLE_DRIVE_FULL_RECONCILIATION_INTERVAL_MS,
|
|
@@ -1377,6 +1534,7 @@ async function resolveKnowledgeSyncProvider(input: {
|
|
|
1377
1534
|
return {
|
|
1378
1535
|
kind: "atlassian",
|
|
1379
1536
|
selectedDestination: source.destination!,
|
|
1537
|
+
aclAuthority: null,
|
|
1380
1538
|
driver: atlassianSyncDriver({
|
|
1381
1539
|
actionProviderCoordinationKey: action.providerCoordinationKey,
|
|
1382
1540
|
metadata,
|
|
@@ -1476,6 +1634,8 @@ export function googleDriveSyncDriver(input: {
|
|
|
1476
1634
|
workspaceId: string;
|
|
1477
1635
|
initiatingSubjectId: string;
|
|
1478
1636
|
authorization: string | undefined;
|
|
1637
|
+
retry: GoogleDriveProviderRetryOptions;
|
|
1638
|
+
observability: Observability;
|
|
1479
1639
|
fullReconciliationIntervalMs: number;
|
|
1480
1640
|
observedExternalObjectIds: (ids: string[]) => Promise<Set<string>>;
|
|
1481
1641
|
provider?: GoogleDriveSyncProviderPort;
|
|
@@ -1490,11 +1650,22 @@ export function googleDriveSyncDriver(input: {
|
|
|
1490
1650
|
const maxElapsedMs = input.limits.maxElapsedSeconds * 1_000;
|
|
1491
1651
|
const provider: GoogleDriveSyncProviderPort = input.provider ?? {
|
|
1492
1652
|
getStartPageToken: async (driveId) =>
|
|
1493
|
-
await getDriveStartPageToken(driveId, input.authorization),
|
|
1653
|
+
await getDriveStartPageToken(driveId, input.authorization, input.retry, input.observability),
|
|
1494
1654
|
listChanges: async (pageToken, driveId, pageSize) =>
|
|
1495
|
-
await listDriveChanges(
|
|
1496
|
-
|
|
1497
|
-
|
|
1655
|
+
await listDriveChanges(
|
|
1656
|
+
pageToken,
|
|
1657
|
+
driveId,
|
|
1658
|
+
pageSize,
|
|
1659
|
+
input.authorization,
|
|
1660
|
+
input.retry,
|
|
1661
|
+
input.observability,
|
|
1662
|
+
),
|
|
1663
|
+
getFile: async (fileId) =>
|
|
1664
|
+
await getDriveFile(fileId, input.authorization, input.retry, input.observability),
|
|
1665
|
+
listChildren: async (request) =>
|
|
1666
|
+
await listDriveChildren(request, input.authorization, input.retry, input.observability),
|
|
1667
|
+
listPermissions: async (request) =>
|
|
1668
|
+
await listDrivePermissions(request, input.authorization, input.retry, input.observability),
|
|
1498
1669
|
};
|
|
1499
1670
|
const clock = provider.now ?? Date.now;
|
|
1500
1671
|
const budgetLimits = {
|
|
@@ -1847,12 +2018,26 @@ export function googleDriveSyncDriver(input: {
|
|
|
1847
2018
|
}
|
|
1848
2019
|
},
|
|
1849
2020
|
fetchContent: async (entry, maxBytes) =>
|
|
1850
|
-
await fetchDriveBytes(entry, input.authorization, maxBytes),
|
|
2021
|
+
await fetchDriveBytes(entry, input.authorization, maxBytes, input.retry, input.observability),
|
|
2022
|
+
readAcl: async (entry, maxProviderRequests) => {
|
|
2023
|
+
if (!provider.listPermissions) {
|
|
2024
|
+
throw new SyncFailure("provider_rejected", false);
|
|
2025
|
+
}
|
|
2026
|
+
return await collectGoogleDriveAclEvidence({
|
|
2027
|
+
entry,
|
|
2028
|
+
maxProviderRequests,
|
|
2029
|
+
listPermissions: provider.listPermissions,
|
|
2030
|
+
getFile: provider.getFile,
|
|
2031
|
+
now: clock,
|
|
2032
|
+
});
|
|
2033
|
+
},
|
|
1851
2034
|
citationLocator: (entry) => ({
|
|
1852
|
-
version:
|
|
2035
|
+
version: 2,
|
|
1853
2036
|
providerKey: "google-drive",
|
|
1854
2037
|
providerCoordinationKey: input.actionProviderCoordinationKey,
|
|
1855
2038
|
externalObjectId: entry.externalObjectId,
|
|
2039
|
+
providerRevision: entry.externalVersionId,
|
|
2040
|
+
driveId: entry.driveId,
|
|
1856
2041
|
sourceUri: entry.sourceUri,
|
|
1857
2042
|
}),
|
|
1858
2043
|
};
|
|
@@ -2180,16 +2365,20 @@ function googleDriveInventoryStopIsHard(
|
|
|
2180
2365
|
async function getDriveStartPageToken(
|
|
2181
2366
|
driveId: string | null,
|
|
2182
2367
|
authorization: string | undefined,
|
|
2368
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2369
|
+
observability: Observability,
|
|
2183
2370
|
): Promise<string> {
|
|
2184
2371
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2185
2372
|
const url = new URL(`${DRIVE_API_BASE}/changes/startPageToken`);
|
|
2186
2373
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2187
2374
|
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2188
|
-
const response = await
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2375
|
+
const response = await requestGoogleDrive(
|
|
2376
|
+
url,
|
|
2377
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2378
|
+
"start_page_token",
|
|
2379
|
+
retry,
|
|
2380
|
+
observability,
|
|
2381
|
+
);
|
|
2193
2382
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2194
2383
|
const payload = (await readResponseJsonBounded(
|
|
2195
2384
|
response,
|
|
@@ -2204,6 +2393,8 @@ async function listDriveChanges(
|
|
|
2204
2393
|
driveId: string | null,
|
|
2205
2394
|
pageSize: number,
|
|
2206
2395
|
authorization: string | undefined,
|
|
2396
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2397
|
+
observability: Observability,
|
|
2207
2398
|
): Promise<GoogleDriveChangesPage> {
|
|
2208
2399
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2209
2400
|
const url = new URL(`${DRIVE_API_BASE}/changes`);
|
|
@@ -2217,11 +2408,13 @@ async function listDriveChanges(
|
|
|
2217
2408
|
"nextPageToken,newStartPageToken,changes(fileId,removed,time,driveId,file(id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed))",
|
|
2218
2409
|
);
|
|
2219
2410
|
if (driveId) url.searchParams.set("driveId", driveId);
|
|
2220
|
-
const response = await
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2411
|
+
const response = await requestGoogleDrive(
|
|
2412
|
+
url,
|
|
2413
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2414
|
+
"list_changes",
|
|
2415
|
+
retry,
|
|
2416
|
+
observability,
|
|
2417
|
+
);
|
|
2225
2418
|
if (response.status === 410) throw new GoogleDriveCursorInvalidError();
|
|
2226
2419
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2227
2420
|
const payload = (await readResponseJsonBounded(
|
|
@@ -2248,6 +2441,8 @@ async function listDriveChanges(
|
|
|
2248
2441
|
async function getDriveFile(
|
|
2249
2442
|
fileId: string,
|
|
2250
2443
|
authorization: string | undefined,
|
|
2444
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2445
|
+
observability: Observability,
|
|
2251
2446
|
): Promise<GoogleDriveInventoryProviderItem | null> {
|
|
2252
2447
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2253
2448
|
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(fileId)}`);
|
|
@@ -2256,11 +2451,13 @@ async function getDriveFile(
|
|
|
2256
2451
|
"fields",
|
|
2257
2452
|
"id,name,mimeType,driveId,parents,modifiedTime,createdTime,version,md5Checksum,size,webViewLink,trashed",
|
|
2258
2453
|
);
|
|
2259
|
-
const response = await
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2454
|
+
const response = await requestGoogleDrive(
|
|
2455
|
+
url,
|
|
2456
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2457
|
+
"get_file",
|
|
2458
|
+
retry,
|
|
2459
|
+
observability,
|
|
2460
|
+
);
|
|
2264
2461
|
if (response.status === 404) return null;
|
|
2265
2462
|
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2266
2463
|
return parseDriveProviderItem(
|
|
@@ -2268,6 +2465,232 @@ async function getDriveFile(
|
|
|
2268
2465
|
);
|
|
2269
2466
|
}
|
|
2270
2467
|
|
|
2468
|
+
async function listDrivePermissions(
|
|
2469
|
+
input: { fileId: string; pageToken: string | null; pageSize: number },
|
|
2470
|
+
authorization: string | undefined,
|
|
2471
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2472
|
+
observability: Observability,
|
|
2473
|
+
): Promise<GoogleDrivePermissionsPage> {
|
|
2474
|
+
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2475
|
+
const url = new URL(`${DRIVE_API_BASE}/files/${encodeURIComponent(input.fileId)}/permissions`);
|
|
2476
|
+
url.searchParams.set("supportsAllDrives", "true");
|
|
2477
|
+
url.searchParams.set("pageSize", String(input.pageSize));
|
|
2478
|
+
url.searchParams.set(
|
|
2479
|
+
"fields",
|
|
2480
|
+
"nextPageToken,permissions(id,type,emailAddress,domain,role,allowFileDiscovery,expirationTime,deleted,permissionDetails(inherited))",
|
|
2481
|
+
);
|
|
2482
|
+
if (input.pageToken) url.searchParams.set("pageToken", input.pageToken);
|
|
2483
|
+
const response = await requestGoogleDrive(
|
|
2484
|
+
url,
|
|
2485
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2486
|
+
"list_permissions",
|
|
2487
|
+
retry,
|
|
2488
|
+
observability,
|
|
2489
|
+
);
|
|
2490
|
+
if (response.status === 403 || response.status === 404) {
|
|
2491
|
+
return { permissions: [], nextPageToken: null, denied: true };
|
|
2492
|
+
}
|
|
2493
|
+
if (!response.ok) throwGoogleDriveResponse(response);
|
|
2494
|
+
const payload = (await readResponseJsonBounded(
|
|
2495
|
+
response,
|
|
2496
|
+
DRIVE_JSON_MAX_BYTES,
|
|
2497
|
+
"Google Drive permission list",
|
|
2498
|
+
)) as Record<string, unknown>;
|
|
2499
|
+
const rawPermissions = Array.isArray(payload.permissions) ? payload.permissions : null;
|
|
2500
|
+
if (!rawPermissions) throw new SyncFailure("provider_payload_invalid", false);
|
|
2501
|
+
return {
|
|
2502
|
+
permissions: rawPermissions.map(parseDrivePermission),
|
|
2503
|
+
nextPageToken: optionalString(payload.nextPageToken),
|
|
2504
|
+
denied: false,
|
|
2505
|
+
};
|
|
2506
|
+
}
|
|
2507
|
+
|
|
2508
|
+
export async function collectGoogleDriveAclEvidence(input: {
|
|
2509
|
+
entry: Pick<GoogleDriveInventoryEntry, "externalObjectId" | "externalVersionId" | "driveId">;
|
|
2510
|
+
maxProviderRequests: number;
|
|
2511
|
+
listPermissions: NonNullable<GoogleDriveSyncProviderPort["listPermissions"]>;
|
|
2512
|
+
getFile?: GoogleDriveSyncProviderPort["getFile"];
|
|
2513
|
+
now?: () => number;
|
|
2514
|
+
}): Promise<KnowledgeSourceSyncAclEvidence> {
|
|
2515
|
+
if (!Number.isSafeInteger(input.maxProviderRequests) || input.maxProviderRequests < 1) {
|
|
2516
|
+
throw new SyncFailure("resource_limit", false);
|
|
2517
|
+
}
|
|
2518
|
+
const now = input.now ?? Date.now;
|
|
2519
|
+
const observedAt = new Date(now());
|
|
2520
|
+
if (!Number.isFinite(observedAt.getTime())) {
|
|
2521
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2522
|
+
}
|
|
2523
|
+
const permissions: GoogleDrivePermission[] = [];
|
|
2524
|
+
const seenTokens = new Set<string>();
|
|
2525
|
+
let pageToken: string | null = null;
|
|
2526
|
+
let providerRequests = 0;
|
|
2527
|
+
let permissionListDenied = false;
|
|
2528
|
+
do {
|
|
2529
|
+
if (providerRequests >= input.maxProviderRequests) {
|
|
2530
|
+
throw new SyncFailure("resource_limit", false);
|
|
2531
|
+
}
|
|
2532
|
+
providerRequests += 1;
|
|
2533
|
+
const page = await input.listPermissions({
|
|
2534
|
+
fileId: input.entry.externalObjectId,
|
|
2535
|
+
pageToken,
|
|
2536
|
+
pageSize: Math.min(100, GOOGLE_DRIVE_ACL_MAX_PRINCIPALS - permissions.length),
|
|
2537
|
+
});
|
|
2538
|
+
if (page.denied) {
|
|
2539
|
+
permissionListDenied = true;
|
|
2540
|
+
break;
|
|
2541
|
+
}
|
|
2542
|
+
permissions.push(...page.permissions);
|
|
2543
|
+
if (permissions.length > GOOGLE_DRIVE_ACL_MAX_PRINCIPALS) {
|
|
2544
|
+
throw new SyncFailure("resource_limit", false);
|
|
2545
|
+
}
|
|
2546
|
+
pageToken = page.nextPageToken;
|
|
2547
|
+
if (pageToken) {
|
|
2548
|
+
if (pageToken.length > 4096 || seenTokens.has(pageToken)) {
|
|
2549
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2550
|
+
}
|
|
2551
|
+
seenTokens.add(pageToken);
|
|
2552
|
+
}
|
|
2553
|
+
} while (pageToken);
|
|
2554
|
+
|
|
2555
|
+
if (permissionListDenied) {
|
|
2556
|
+
if (!input.getFile || providerRequests >= input.maxProviderRequests) {
|
|
2557
|
+
return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
|
|
2558
|
+
}
|
|
2559
|
+
providerRequests += 1;
|
|
2560
|
+
const current = await input.getFile(input.entry.externalObjectId);
|
|
2561
|
+
if (!current || current.trashed) {
|
|
2562
|
+
return googleDriveDeniedAclEvidence(input.entry, observedAt, providerRequests);
|
|
2563
|
+
}
|
|
2564
|
+
const aclRevision = createHash("sha256")
|
|
2565
|
+
.update(
|
|
2566
|
+
JSON.stringify({
|
|
2567
|
+
version: 1,
|
|
2568
|
+
mode: "source_owner_only",
|
|
2569
|
+
externalObjectId: input.entry.externalObjectId,
|
|
2570
|
+
providerRevision: input.entry.externalVersionId,
|
|
2571
|
+
}),
|
|
2572
|
+
)
|
|
2573
|
+
.digest("hex");
|
|
2574
|
+
return {
|
|
2575
|
+
eligibility: "eligible",
|
|
2576
|
+
providerRevision: input.entry.externalVersionId,
|
|
2577
|
+
driveId: input.entry.driveId,
|
|
2578
|
+
aclRevision,
|
|
2579
|
+
observedAt: observedAt.toISOString(),
|
|
2580
|
+
expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
|
|
2581
|
+
providerRequests,
|
|
2582
|
+
principals: [],
|
|
2583
|
+
};
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
const canonicalPermissions = permissions
|
|
2587
|
+
.map((permission) => ({
|
|
2588
|
+
id: permission.id,
|
|
2589
|
+
type: permission.type,
|
|
2590
|
+
role: permission.role,
|
|
2591
|
+
emailAddress: permission.emailAddress,
|
|
2592
|
+
domain: permission.domain,
|
|
2593
|
+
allowFileDiscovery: permission.allowFileDiscovery,
|
|
2594
|
+
expirationTime: permission.expirationTime,
|
|
2595
|
+
inherited: permission.inherited,
|
|
2596
|
+
}))
|
|
2597
|
+
.sort((left, right) => JSON.stringify(left).localeCompare(JSON.stringify(right)));
|
|
2598
|
+
const aclRevision = createHash("sha256")
|
|
2599
|
+
.update(
|
|
2600
|
+
JSON.stringify({
|
|
2601
|
+
version: 1,
|
|
2602
|
+
externalObjectId: input.entry.externalObjectId,
|
|
2603
|
+
providerRevision: input.entry.externalVersionId,
|
|
2604
|
+
permissions: canonicalPermissions,
|
|
2605
|
+
}),
|
|
2606
|
+
)
|
|
2607
|
+
.digest("hex");
|
|
2608
|
+
return {
|
|
2609
|
+
eligibility: "eligible",
|
|
2610
|
+
providerRevision: input.entry.externalVersionId,
|
|
2611
|
+
driveId: input.entry.driveId,
|
|
2612
|
+
aclRevision,
|
|
2613
|
+
observedAt: observedAt.toISOString(),
|
|
2614
|
+
expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
|
|
2615
|
+
providerRequests,
|
|
2616
|
+
principals: canonicalPermissions.map((permission) => ({
|
|
2617
|
+
type: permission.type,
|
|
2618
|
+
permissionId: permission.id,
|
|
2619
|
+
emailAddress: permission.emailAddress,
|
|
2620
|
+
domain: permission.domain,
|
|
2621
|
+
role: permission.role,
|
|
2622
|
+
inherited: permission.inherited,
|
|
2623
|
+
allowFileDiscovery: permission.allowFileDiscovery,
|
|
2624
|
+
expirationTime: permission.expirationTime,
|
|
2625
|
+
})),
|
|
2626
|
+
};
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
function googleDriveDeniedAclEvidence(
|
|
2630
|
+
entry: Pick<GoogleDriveInventoryEntry, "externalObjectId" | "externalVersionId" | "driveId">,
|
|
2631
|
+
observedAt: Date,
|
|
2632
|
+
providerRequests: number,
|
|
2633
|
+
) {
|
|
2634
|
+
return {
|
|
2635
|
+
eligibility: "denied" as const,
|
|
2636
|
+
providerRevision: entry.externalVersionId,
|
|
2637
|
+
driveId: entry.driveId,
|
|
2638
|
+
aclRevision: createHash("sha256")
|
|
2639
|
+
.update(
|
|
2640
|
+
JSON.stringify({
|
|
2641
|
+
version: 1,
|
|
2642
|
+
mode: "provider_denied",
|
|
2643
|
+
externalObjectId: entry.externalObjectId,
|
|
2644
|
+
providerRevision: entry.externalVersionId,
|
|
2645
|
+
}),
|
|
2646
|
+
)
|
|
2647
|
+
.digest("hex"),
|
|
2648
|
+
observedAt: observedAt.toISOString(),
|
|
2649
|
+
expiresAt: new Date(observedAt.getTime() + GOOGLE_DRIVE_ACL_FRESHNESS_MS).toISOString(),
|
|
2650
|
+
providerRequests,
|
|
2651
|
+
principals: [],
|
|
2652
|
+
};
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
function parseDrivePermission(value: unknown): GoogleDrivePermission {
|
|
2656
|
+
const row = objectRecord(value);
|
|
2657
|
+
if (row.deleted === true) throw new SyncFailure("provider_payload_invalid", false);
|
|
2658
|
+
const type = requiredString(row.type, "permission.type");
|
|
2659
|
+
if (type !== "user" && type !== "group" && type !== "domain" && type !== "anyone") {
|
|
2660
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2661
|
+
}
|
|
2662
|
+
const role = requiredString(row.role, "permission.role");
|
|
2663
|
+
if (
|
|
2664
|
+
role !== "owner" &&
|
|
2665
|
+
role !== "organizer" &&
|
|
2666
|
+
role !== "fileOrganizer" &&
|
|
2667
|
+
role !== "writer" &&
|
|
2668
|
+
role !== "commenter" &&
|
|
2669
|
+
role !== "reader"
|
|
2670
|
+
) {
|
|
2671
|
+
throw new SyncFailure("provider_payload_invalid", false);
|
|
2672
|
+
}
|
|
2673
|
+
const permissionDetails = Array.isArray(row.permissionDetails) ? row.permissionDetails : [];
|
|
2674
|
+
return {
|
|
2675
|
+
id: requiredString(row.id, "permission.id"),
|
|
2676
|
+
type,
|
|
2677
|
+
role,
|
|
2678
|
+
emailAddress: optionalString(row.emailAddress),
|
|
2679
|
+
domain: optionalString(row.domain),
|
|
2680
|
+
allowFileDiscovery: typeof row.allowFileDiscovery === "boolean" ? row.allowFileDiscovery : null,
|
|
2681
|
+
expirationTime: optionalExactProviderTimestamp(row.expirationTime),
|
|
2682
|
+
inherited: permissionDetails.some((detail) => objectRecord(detail).inherited === true),
|
|
2683
|
+
};
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
function optionalExactProviderTimestamp(value: unknown): string | null {
|
|
2687
|
+
const timestamp = optionalString(value);
|
|
2688
|
+
if (!timestamp) return null;
|
|
2689
|
+
const parsed = new Date(timestamp);
|
|
2690
|
+
if (!Number.isFinite(parsed.getTime())) throw new SyncFailure("provider_payload_invalid", false);
|
|
2691
|
+
return parsed.toISOString();
|
|
2692
|
+
}
|
|
2693
|
+
|
|
2271
2694
|
function throwGoogleDriveResponse(response: Response): never {
|
|
2272
2695
|
if (response.status === 401 || response.status === 403) {
|
|
2273
2696
|
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
@@ -2280,6 +2703,31 @@ function throwGoogleDriveResponse(response: Response): never {
|
|
|
2280
2703
|
);
|
|
2281
2704
|
}
|
|
2282
2705
|
|
|
2706
|
+
async function requestGoogleDrive(
|
|
2707
|
+
url: URL,
|
|
2708
|
+
init: RequestInit,
|
|
2709
|
+
operation: GoogleDriveProviderOperation,
|
|
2710
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2711
|
+
observability: Observability,
|
|
2712
|
+
): Promise<Response> {
|
|
2713
|
+
try {
|
|
2714
|
+
return await fetchGoogleDriveProvider({
|
|
2715
|
+
fetchImpl: fetch,
|
|
2716
|
+
url,
|
|
2717
|
+
init,
|
|
2718
|
+
operation,
|
|
2719
|
+
policy: retry,
|
|
2720
|
+
observability,
|
|
2721
|
+
heartbeat: (details) => heartbeat({ provider: "google_drive", ...details }),
|
|
2722
|
+
});
|
|
2723
|
+
} catch (error) {
|
|
2724
|
+
if (error instanceof GoogleDriveProviderTransportError) {
|
|
2725
|
+
throw new SyncFailure("provider_unavailable", true);
|
|
2726
|
+
}
|
|
2727
|
+
throw error;
|
|
2728
|
+
}
|
|
2729
|
+
}
|
|
2730
|
+
|
|
2283
2731
|
function selectedDriveSource(
|
|
2284
2732
|
metadata: ReturnType<typeof GoogleDriveConnectionMetadata.parse>,
|
|
2285
2733
|
externalSourceId: string,
|
|
@@ -2308,6 +2756,8 @@ export function googleDriveSyncProviderAccessAllowed(input: {
|
|
|
2308
2756
|
async function listDriveChildren(
|
|
2309
2757
|
input: { folderId: string; driveId: string | null; pageToken: string | null; pageSize: number },
|
|
2310
2758
|
authorization: string | undefined,
|
|
2759
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2760
|
+
observability: Observability,
|
|
2311
2761
|
): Promise<GoogleDriveInventoryPage> {
|
|
2312
2762
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2313
2763
|
const url = new URL(`${DRIVE_API_BASE}/files`);
|
|
@@ -2327,11 +2777,13 @@ async function listDriveChildren(
|
|
|
2327
2777
|
url.searchParams.set("corpora", "drive");
|
|
2328
2778
|
url.searchParams.set("driveId", input.driveId);
|
|
2329
2779
|
}
|
|
2330
|
-
const response = await
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2780
|
+
const response = await requestGoogleDrive(
|
|
2781
|
+
url,
|
|
2782
|
+
{ headers: { authorization, accept: "application/json" } },
|
|
2783
|
+
"list_children",
|
|
2784
|
+
retry,
|
|
2785
|
+
observability,
|
|
2786
|
+
);
|
|
2335
2787
|
if (!response.ok) {
|
|
2336
2788
|
if (response.status === 401 || response.status === 403) {
|
|
2337
2789
|
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
@@ -2378,6 +2830,8 @@ async function fetchDriveBytes(
|
|
|
2378
2830
|
entry: GoogleDriveInventoryEntry,
|
|
2379
2831
|
authorization: string | undefined,
|
|
2380
2832
|
maxBytes: number,
|
|
2833
|
+
retry: GoogleDriveProviderRetryOptions,
|
|
2834
|
+
observability: Observability,
|
|
2381
2835
|
): Promise<Uint8Array> {
|
|
2382
2836
|
if (!authorization) throw new SyncFailure("connection_reconnect_required", true, true);
|
|
2383
2837
|
const url =
|
|
@@ -2390,11 +2844,13 @@ async function fetchDriveBytes(
|
|
|
2390
2844
|
url.searchParams.set("alt", "media");
|
|
2391
2845
|
url.searchParams.set("supportsAllDrives", "true");
|
|
2392
2846
|
}
|
|
2393
|
-
const response = await
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2847
|
+
const response = await requestGoogleDrive(
|
|
2848
|
+
url,
|
|
2849
|
+
{ headers: { authorization } },
|
|
2850
|
+
entry.transfer.action === "export" ? "export" : "download",
|
|
2851
|
+
retry,
|
|
2852
|
+
observability,
|
|
2853
|
+
);
|
|
2398
2854
|
if (!response.ok) {
|
|
2399
2855
|
if (response.status === 401 || response.status === 403) {
|
|
2400
2856
|
throw new SyncFailure("connection_reconnect_required", true, true);
|
|
@@ -2826,6 +3282,66 @@ function inventoryLimit(
|
|
|
2826
3282
|
return null;
|
|
2827
3283
|
}
|
|
2828
3284
|
|
|
3285
|
+
function recordSyncHealthTelemetry(
|
|
3286
|
+
observability: Observability,
|
|
3287
|
+
provider: "google_drive" | "atlassian",
|
|
3288
|
+
summary: KnowledgeSourceSyncRunSummary,
|
|
3289
|
+
outcome: "succeeded" | "failed",
|
|
3290
|
+
failure?: SyncFailure,
|
|
3291
|
+
): void {
|
|
3292
|
+
observability.observeHistogram({
|
|
3293
|
+
name: "opengeni_knowledge_source_sync_terminal_batch_duration_seconds",
|
|
3294
|
+
help: "Terminal knowledge-source activity batch duration by provider and outcome.",
|
|
3295
|
+
labels: { provider, outcome },
|
|
3296
|
+
value: Math.max(0, summary.elapsedMs) / 1_000,
|
|
3297
|
+
});
|
|
3298
|
+
if (summary.providerRequests > 0) {
|
|
3299
|
+
observability.incrementCounter({
|
|
3300
|
+
name: "opengeni_knowledge_source_sync_provider_requests_total",
|
|
3301
|
+
help: "Logical provider requests consumed by knowledge-source synchronization.",
|
|
3302
|
+
labels: { provider, outcome },
|
|
3303
|
+
amount: summary.providerRequests,
|
|
3304
|
+
});
|
|
3305
|
+
}
|
|
3306
|
+
if (summary.limitReached) {
|
|
3307
|
+
observability.incrementCounter({
|
|
3308
|
+
name: "opengeni_knowledge_source_sync_limit_hits_total",
|
|
3309
|
+
help: "Knowledge-source synchronization runs stopped by an explicit resource budget.",
|
|
3310
|
+
labels: { provider, limit: summary.limitReached },
|
|
3311
|
+
});
|
|
3312
|
+
}
|
|
3313
|
+
if (summary.reconnectRequired) {
|
|
3314
|
+
observability.incrementCounter({
|
|
3315
|
+
name: "opengeni_knowledge_source_sync_reconnect_required_total",
|
|
3316
|
+
help: "Knowledge-source synchronization failures requiring connector reconnection.",
|
|
3317
|
+
labels: { provider },
|
|
3318
|
+
});
|
|
3319
|
+
}
|
|
3320
|
+
if (failure) {
|
|
3321
|
+
observability.incrementCounter({
|
|
3322
|
+
name: "opengeni_knowledge_source_sync_failures_total",
|
|
3323
|
+
help: "Knowledge-source synchronization terminal failures by bounded reason.",
|
|
3324
|
+
labels: {
|
|
3325
|
+
provider,
|
|
3326
|
+
reason: failure.code,
|
|
3327
|
+
retryable: failure.retryable ? "true" : "false",
|
|
3328
|
+
},
|
|
3329
|
+
});
|
|
3330
|
+
observability.warn("Knowledge source sync failed", {
|
|
3331
|
+
provider,
|
|
3332
|
+
outcome: failure.retryable ? "retryable_failed" : "failed",
|
|
3333
|
+
reason: failure.code,
|
|
3334
|
+
durationMs: summary.elapsedMs,
|
|
3335
|
+
});
|
|
3336
|
+
return;
|
|
3337
|
+
}
|
|
3338
|
+
observability.info("Knowledge source sync completed", {
|
|
3339
|
+
provider,
|
|
3340
|
+
outcome: "succeeded",
|
|
3341
|
+
durationMs: summary.elapsedMs,
|
|
3342
|
+
});
|
|
3343
|
+
}
|
|
3344
|
+
|
|
2829
3345
|
type SyncFailureCode = KnowledgeSourceSyncRunSummary["failures"][number]["code"];
|
|
2830
3346
|
|
|
2831
3347
|
class SyncFailure extends Error {
|