@lobu/runtime-server 20.2.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/connectors/README.md +15 -16
- package/dist/connectors/github.ts +15 -9
- package/dist/connectors/google_calendar.ts +21 -31
- package/dist/connectors/google_drive.ts +106 -131
- package/dist/connectors/google_gmail.ts +53 -30
- package/dist/connectors/hackernews.ts +2 -5
- package/dist/connectors/jira.ts +62 -27
- package/dist/connectors/linear.ts +55 -21
- package/dist/connectors/microsoft_outlook.ts +151 -110
- package/dist/connectors/postgres.ts +19 -11
- package/dist/connectors/producthunt.ts +8 -6
- package/dist/connectors/reddit.ts +2 -2
- package/dist/connectors/rss.ts +2 -2
- package/dist/connectors/whatsapp_web.ts +5 -4
- package/dist/connectors/x.ts +32 -17
- package/dist/connectors/youtube.ts +18 -5
- package/dist/db/migrations/20260917000000_views.sql +271 -0
- package/dist/db/migrations/20260917000001_views_attach_index.sql +14 -0
- package/dist/db/migrations/20260918000000_runs_turn_tool_event_uniq.sql +51 -0
- package/dist/db/migrations/20260923230000_runs_connector_artifact_hash.sql +32 -0
- package/dist/db/migrations/20260924000000_resource_source_files.sql +17 -0
- package/dist/dependencies.bun.lock +109 -98
- package/dist/guest.bundle.js +78 -10
- package/dist/owletto/dist/assets/_-CwFknmeA.js +1 -0
- package/dist/owletto/dist/assets/_-DrcppoWh.js +2 -0
- package/dist/owletto/dist/assets/_agentId-Cr57JKYx.js +1 -0
- package/dist/owletto/dist/assets/_agentId-HNcmgH56.js +1 -0
- package/dist/owletto/dist/assets/_automationId-BCfrLnUS.js +1 -0
- package/dist/owletto/dist/assets/{_connectorKey._connectionId-BAJGfcy7.js → _connectorKey._connectionId-126-iG9w.js} +1 -1
- package/dist/owletto/dist/assets/{_connectorKey.index-BBF8vGJS.js → _connectorKey.index-CywE9A3N.js} +1 -1
- package/dist/owletto/dist/assets/_conversationId-CNETA_rY.js +1 -0
- package/dist/owletto/dist/assets/_deploymentId-CAVpXf8s.js +1 -0
- package/dist/owletto/dist/assets/_name-BP7OlVxL.js +1 -0
- package/dist/owletto/dist/assets/_owner-D2eh4R90.js +1 -0
- package/dist/owletto/dist/assets/_shape-BqV8tSfw.js +1 -0
- package/dist/owletto/dist/assets/_skillKey-2xTjhL_C.js +1 -0
- package/dist/owletto/dist/assets/_slug-DoIaJBbh.js +1 -0
- package/dist/owletto/dist/assets/_viewKey-8ePnWEg-.js +1 -0
- package/dist/owletto/dist/assets/_viewKey-D8rjUFLV.js +1 -0
- package/dist/owletto/dist/assets/_viewKey-DhVOkzYd.js +1 -0
- package/dist/owletto/dist/assets/accept-invitation-BWRiEUsd.js +1 -0
- package/dist/owletto/dist/assets/activity-route-view-rVdTeVXN.js +1 -0
- package/dist/owletto/dist/assets/{agent-card-controls-CZO6GI0s.js → agent-card-controls-JubkhcHt.js} +1 -1
- package/dist/owletto/dist/assets/{agent-client-setup-list-_chLOorY.js → agent-client-setup-list-Btcy4Cu1.js} +1 -1
- package/dist/owletto/dist/assets/{agent-connect-pages-DWBgSIYT.js → agent-connect-pages-Chfqn8iL.js} +1 -1
- package/dist/owletto/dist/assets/{agent-editor-form-BTNlg0kQ.js → agent-editor-form-BrOkwfg8.js} +1 -1
- package/dist/owletto/dist/assets/{agent-guardrails-route-B5uoMC6o.js → agent-guardrails-route-CHM7JbIl.js} +1 -1
- package/dist/owletto/dist/assets/{agent-loading-shells-CLoLeFcB.js → agent-loading-shells-DsiGx1fn.js} +1 -1
- package/dist/owletto/dist/assets/{agent-permissions-matrix-TP00-MYL.js → agent-permissions-matrix-L_gFvSu1.js} +1 -1
- package/dist/owletto/dist/assets/{agent-skill-context-D4a2QCoZ.js → agent-skill-context-D7tOJ8Pb.js} +1 -1
- package/dist/owletto/dist/assets/{agent-skills-route-KVDDTbJB.js → agent-skills-route-AylS-B9v.js} +1 -1
- package/dist/owletto/dist/assets/{agent-subpage-shell-DuVYZvBE.js → agent-subpage-shell-B7tCQ9zd.js} +1 -1
- package/dist/owletto/dist/assets/agent-thread-DOYh7WLW.js +8 -0
- package/dist/owletto/dist/assets/agents-0ncYwbz6.js +1 -0
- package/dist/owletto/dist/assets/agents-workbench-DOwn78eL.js +1 -0
- package/dist/owletto/dist/assets/app-bridge-BH3g2mKG.js +127 -0
- package/dist/owletto/dist/assets/{app._appId-D0_LQTJU.js → app._appId-fJ_RRn1Z.js} +1 -1
- package/dist/owletto/dist/assets/automation-detail-content-CMprQht5.js +1 -0
- package/dist/owletto/dist/assets/automations-0ncYwbz6.js +1 -0
- package/dist/owletto/dist/assets/{calendar-C0XpOHd8.js → calendar-BbD1N97u.js} +1 -1
- package/dist/owletto/dist/assets/{callback-CO_K9eR-.js → callback-PELM84Lh.js} +1 -1
- package/dist/owletto/dist/assets/{card-DeFQuU_J.js → card-DpLjEqIc.js} +1 -1
- package/dist/owletto/dist/assets/{claim-CUpWOSie.js → claim-BLLvLOBM.js} +1 -1
- package/dist/owletto/dist/assets/clients-Bd8WpHSV.js +1 -0
- package/dist/owletto/dist/assets/{code-editor-BqGUZFt6.js → code-editor-DGJQDoHD.js} +1 -1
- package/dist/owletto/dist/assets/connect-Cx49f4J0.js +1 -0
- package/dist/owletto/dist/assets/connect._clientId-BDFBqKSA.js +1 -0
- package/dist/owletto/dist/assets/connected._clientId-BYGmFRCF.js +1 -0
- package/dist/owletto/dist/assets/{connection-BgAlYjeV.js → connection-B0O8OZuK.js} +1 -1
- package/dist/owletto/dist/assets/{connection-feed-detail-B3ePAinf.js → connection-feed-detail-DZjtoEfm.js} +1 -1
- package/dist/owletto/dist/assets/{connector-provenance-DgsDg0Z9.js → connector-provenance-D_g9gGXU.js} +1 -1
- package/dist/owletto/dist/assets/{connectors-CqPQPo9U.js → connectors-BiMMm_QQ.js} +1 -1
- package/dist/owletto/dist/assets/{consent-bswcIzsf.js → consent-CIzW-960.js} +1 -1
- package/dist/owletto/dist/assets/{core.esm-DsLgjoel.js → core.esm-Ds4Nuq9d.js} +1 -1
- package/dist/owletto/dist/assets/create--fsxroj2.js +1 -0
- package/dist/owletto/dist/assets/create-BqL-ahEf.js +1 -0
- package/dist/owletto/dist/assets/{create-Dwo04oWV.js → create-BvCaqhwK.js} +1 -1
- package/dist/owletto/dist/assets/create-automation-form-DN2w080o.js +2 -0
- package/dist/owletto/dist/assets/{create._connectorKey-CCBb2pil.js → create._connectorKey--qDi1h6S.js} +1 -1
- package/dist/owletto/dist/assets/{cronstrue-ChFyC9ps.js → cronstrue-h8_zIyZD.js} +1 -1
- package/dist/owletto/dist/assets/deployments-BROg1Te8.js +1 -0
- package/dist/owletto/dist/assets/deployments-ICDss7yS.js +1 -0
- package/dist/owletto/dist/assets/deployments-body-DulCUAMt.js +1 -0
- package/dist/owletto/dist/assets/device-kTL2l_cz.js +1 -0
- package/dist/owletto/dist/assets/{device-pin-tombstone-1ikyqSdb.js → device-pin-tombstone-BmD4pv_e.js} +1 -1
- package/dist/owletto/dist/assets/device-rows-BqudjhFu.js +1 -0
- package/dist/owletto/dist/assets/{device._deviceId-DTYKGBY8.js → device._deviceId-CsLJJ6jv.js} +1 -1
- package/dist/owletto/dist/assets/entity-options-ErxYvMDH.js +1 -0
- package/dist/owletto/dist/assets/entity-page-CYRAwUGO.js +1 -0
- package/dist/owletto/dist/assets/entity-type-page-BlAizGlz.js +1 -0
- package/dist/owletto/dist/assets/event-page-DTXPxc-E.js +1 -0
- package/dist/owletto/dist/assets/events-feed-L3c-LLiJ.js +1 -0
- package/dist/owletto/dist/assets/{full-page-spinner-BN2G_YiM.js → full-page-spinner-BxcYpJXx.js} +1 -1
- package/dist/owletto/dist/assets/guardrails-Dh4adTXY.js +1 -0
- package/dist/owletto/dist/assets/identity-B8lKn0je.js +1 -0
- package/dist/owletto/dist/assets/{index-C6VAJ8Ku.js → index--lEHrFE4.js} +1 -1
- package/dist/owletto/dist/assets/{index-hSE2E46p.js → index-13rywWam.js} +1 -1
- package/dist/owletto/dist/assets/{index-C-u4yR8Z.js → index-BRvT0VV9.js} +1 -1
- package/dist/owletto/dist/assets/index-BUp24ZJr.js +1 -0
- package/dist/owletto/dist/assets/index-B_-MLsof.js +1 -0
- package/dist/owletto/dist/assets/index-Bq-q4Y7E.js +1 -0
- package/dist/owletto/dist/assets/index-C8CR2-Ck.css +1 -0
- package/dist/owletto/dist/assets/{index-OUb5Mlez.js → index-CDz3-ieP.js} +10 -10
- package/dist/owletto/dist/assets/index-CFkj9iR_.js +1 -0
- package/dist/owletto/dist/assets/index-CgBgOElA.js +1 -0
- package/dist/owletto/dist/assets/{index-C2oCgDX7.js → index-CheCJomo.js} +1 -1
- package/dist/owletto/dist/assets/index-CpiW66xK.js +1 -0
- package/dist/owletto/dist/assets/index-DAv5S2aK.js +1 -0
- package/dist/owletto/dist/assets/index-DBi2Xr3i.js +1 -0
- package/dist/owletto/dist/assets/index-DHOha5OJ.js +1 -0
- package/dist/owletto/dist/assets/index-DdGjY1Hb.js +7898 -0
- package/dist/owletto/dist/assets/index-Dkyl7l6E.js +1 -0
- package/dist/owletto/dist/assets/{index-vdqhlocx.js → index-QcHQZI87.js} +1 -1
- package/dist/owletto/dist/assets/index-kq4537-S.js +1 -0
- package/dist/owletto/dist/assets/{json-diff-DnsW7Pin.js → json-diff-BV2STDGj.js} +2 -2
- package/dist/owletto/dist/assets/{list-skeleton-CHzLN7GK.js → list-skeleton-CT5n4ND_.js} +1 -1
- package/dist/owletto/dist/assets/login-DKsgABYt.js +1 -0
- package/dist/owletto/dist/assets/logout-D_CP-emw.js +1 -0
- package/dist/owletto/dist/assets/member-scope-gate-BTgmlfFJ.js +1 -0
- package/dist/owletto/dist/assets/memory-browse-page-FHaJC3ML.js +101 -0
- package/dist/owletto/dist/assets/missing-view-Bve6pJP-.js +1 -0
- package/dist/owletto/dist/assets/{model-selector-DfbmHjLu.js → model-selector-BLCEahny.js} +1 -1
- package/dist/owletto/dist/assets/models-C4N7LQAW.js +1 -0
- package/dist/owletto/dist/assets/new-BAIGSdfp.js +1 -0
- package/dist/owletto/dist/assets/{new-Df_dY6Z5.js → new-CirOK0Sw.js} +1 -1
- package/dist/owletto/dist/assets/new-DzapH2sV.js +1 -0
- package/dist/owletto/dist/assets/{oauth-workspace-access-D9GmiMfK.js → oauth-workspace-access-BF7jaswv.js} +1 -1
- package/dist/owletto/dist/assets/{organization-uSDYkJpX.js → organization-D9RPsWG-.js} +1 -1
- package/dist/owletto/dist/assets/{permissions-Fe0DP6b8.js → permissions-BlRONw-u.js} +1 -1
- package/dist/owletto/dist/assets/{proposal-review-shell-D9HwnDda.js → proposal-review-shell-uskeFIUr.js} +1 -1
- package/dist/owletto/dist/assets/{recent-C03LhMD7.js → recent-CmJ4Is6V.js} +1 -1
- package/dist/owletto/dist/assets/{renderer-CHNejd95.js → renderer-Dyf4t8kR.js} +2 -2
- package/dist/owletto/dist/assets/{reset-password-CuISnlG_.js → reset-password-BVIeE1Zm.js} +1 -1
- package/dist/owletto/dist/assets/runs-CzBeUzlQ.js +1 -0
- package/dist/owletto/dist/assets/{runs-panel-DsdESupx.js → runs-panel-D7l1uLGm.js} +1 -1
- package/dist/owletto/dist/assets/runs-table-dATyiBg2.js +1 -0
- package/dist/owletto/dist/assets/{schema-value-normalization-B--OWDL8.js → schema-value-normalization-DgMkyVVE.js} +1 -1
- package/dist/owletto/dist/assets/settings-BK1GHmzb.js +1 -0
- package/dist/owletto/dist/assets/{settings-wGNHtsDP.js → settings-Co1wj_m6.js} +1 -1
- package/dist/owletto/dist/assets/skills-DF53JcYt.js +1 -0
- package/dist/owletto/dist/assets/{switch-Dzse_XFM.js → switch-DDkPrV3p.js} +1 -1
- package/dist/owletto/dist/assets/use-applied-views-DELwQQh7.js +1 -0
- package/dist/owletto/dist/assets/{use-mention-search-uyW1Uwf6.js → use-mention-search-CGA3AaKm.js} +1 -1
- package/dist/owletto/dist/assets/view-frame-DDr-Xtet.js +1 -0
- package/dist/owletto/dist/index.html +2 -2
- package/dist/server-main.bundle.mjs +6703 -3581
- package/npm-shrinkwrap.json +530 -472
- package/package.json +5 -2
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
- package/vendor/core/dist/contracts/worker/protocol.js +58 -9
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +25 -5
- package/vendor/views/dist/api.d.ts +179 -0
- package/vendor/views/dist/api.js +406 -0
- package/vendor/views/dist/bridge.d.ts +141 -0
- package/vendor/views/dist/bridge.js +330 -0
- package/vendor/views/dist/index.d.ts +12 -0
- package/vendor/views/dist/index.js +10 -0
- package/vendor/views/package.json +35 -0
- package/dist/owletto/dist/assets/_-Ds4DyQjj.js +0 -2
- package/dist/owletto/dist/assets/_-VmyR6aN_.js +0 -1
- package/dist/owletto/dist/assets/_agentId-BAVnp0c7.js +0 -1
- package/dist/owletto/dist/assets/_agentId-Df-3rGi5.js +0 -1
- package/dist/owletto/dist/assets/_automationId-D696vOjd.js +0 -1
- package/dist/owletto/dist/assets/_conversationId-Ci4rgrHm.js +0 -1
- package/dist/owletto/dist/assets/_deploymentId-Iy-D5PDq.js +0 -1
- package/dist/owletto/dist/assets/_name-CeGwb0oK.js +0 -1
- package/dist/owletto/dist/assets/_owner-CFR1G8OX.js +0 -1
- package/dist/owletto/dist/assets/_skillKey-DuLcQWBu.js +0 -1
- package/dist/owletto/dist/assets/_slug-CDSO20ac.js +0 -1
- package/dist/owletto/dist/assets/accept-invitation-Db5jklHT.js +0 -1
- package/dist/owletto/dist/assets/activity-route-view-DVW3DoB7.js +0 -1
- package/dist/owletto/dist/assets/agent-thread-CzCJU3-R.js +0 -134
- package/dist/owletto/dist/assets/agents-DFs57C3c.js +0 -1
- package/dist/owletto/dist/assets/agents-workbench-C2bUlJP2.js +0 -1
- package/dist/owletto/dist/assets/automation-detail-content-DWkdhnv2.js +0 -1
- package/dist/owletto/dist/assets/automations-DFs57C3c.js +0 -1
- package/dist/owletto/dist/assets/clients-DhYr4lyL.js +0 -1
- package/dist/owletto/dist/assets/connect-Dtb-E5An.js +0 -1
- package/dist/owletto/dist/assets/connect._clientId-DWtceAAw.js +0 -1
- package/dist/owletto/dist/assets/connected._clientId-CMe-wTuV.js +0 -1
- package/dist/owletto/dist/assets/create-CcXUdke7.js +0 -1
- package/dist/owletto/dist/assets/create-Cp6hMoeU.js +0 -1
- package/dist/owletto/dist/assets/create-automation-form-CJ4meayB.js +0 -1
- package/dist/owletto/dist/assets/deployments-CjfFxyDA.js +0 -1
- package/dist/owletto/dist/assets/deployments-body-Cg0KiWGr.js +0 -1
- package/dist/owletto/dist/assets/device-DpY67Jaj.js +0 -1
- package/dist/owletto/dist/assets/device-rows-Cp3yMcuV.js +0 -1
- package/dist/owletto/dist/assets/entity-options-1EaSGMX4.js +0 -1
- package/dist/owletto/dist/assets/entity-page-DJmM70dv.js +0 -1
- package/dist/owletto/dist/assets/entity-type-page-C3mT7tXn.js +0 -1
- package/dist/owletto/dist/assets/events-feed-iliGfjit.js +0 -1
- package/dist/owletto/dist/assets/guardrails-CLlnRRy9.js +0 -1
- package/dist/owletto/dist/assets/identity-MSLryUi8.js +0 -1
- package/dist/owletto/dist/assets/index-3o4Eepb4.js +0 -1
- package/dist/owletto/dist/assets/index-BUr8321-.js +0 -7898
- package/dist/owletto/dist/assets/index-Bulu5ldM.js +0 -1
- package/dist/owletto/dist/assets/index-BvbfITqH.js +0 -1
- package/dist/owletto/dist/assets/index-ChcQMdod.css +0 -1
- package/dist/owletto/dist/assets/index-Clw9ZBj-.js +0 -1
- package/dist/owletto/dist/assets/index-D2oL4Ydo.js +0 -1
- package/dist/owletto/dist/assets/index-D7wulT4y.js +0 -1
- package/dist/owletto/dist/assets/index-DOLFe19K.js +0 -1
- package/dist/owletto/dist/assets/index-DTSA0Xh1.js +0 -1
- package/dist/owletto/dist/assets/index-DxXXVPU0.js +0 -1
- package/dist/owletto/dist/assets/index-nzvsA3hT.js +0 -1
- package/dist/owletto/dist/assets/login-CCKNM0gV.js +0 -1
- package/dist/owletto/dist/assets/logout-B-N2aodZ.js +0 -1
- package/dist/owletto/dist/assets/member-scope-gate-B560Wg-t.js +0 -1
- package/dist/owletto/dist/assets/memory-browse-page-DMDqKJn7.js +0 -101
- package/dist/owletto/dist/assets/models-M51OGDRE.js +0 -1
- package/dist/owletto/dist/assets/new-BVQS9H-Y.js +0 -1
- package/dist/owletto/dist/assets/new-DxDVE9nL.js +0 -1
- package/dist/owletto/dist/assets/runs-BnYdToDc.js +0 -1
- package/dist/owletto/dist/assets/runs-table-B7jKuILd.js +0 -1
- package/dist/owletto/dist/assets/settings-Bg9yCCoN.js +0 -1
- package/dist/owletto/dist/assets/skills-CN5Ube7A.js +0 -1
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
package/dist/component.json
CHANGED
|
@@ -44,11 +44,8 @@ export default defineConnector({
|
|
|
44
44
|
// Fetch data, transform to events...
|
|
45
45
|
const events: EventEnvelope[] = [];
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
checkpoint: { last_sync_at: new Date().toISOString() },
|
|
50
|
-
metadata: { query, items_found: events.length },
|
|
51
|
-
};
|
|
47
|
+
await ctx.commit(events, { last_sync_at: new Date().toISOString() });
|
|
48
|
+
return { status: 'complete', metadata: { query, items_found: events.length } };
|
|
52
49
|
},
|
|
53
50
|
},
|
|
54
51
|
},
|
|
@@ -234,10 +231,11 @@ feeds: {
|
|
|
234
231
|
issue: { description: 'An issue changed' },
|
|
235
232
|
},
|
|
236
233
|
// Incremental materialization for local search and Automations.
|
|
237
|
-
sync: async (ctx) =>
|
|
238
|
-
events
|
|
239
|
-
|
|
240
|
-
|
|
234
|
+
sync: async (ctx) => {
|
|
235
|
+
const events = await fetchChangedIssues(ctx.config.project, ctx.checkpoint);
|
|
236
|
+
await ctx.commit(events, { synced_at: new Date().toISOString() });
|
|
237
|
+
return { status: 'complete' };
|
|
238
|
+
},
|
|
241
239
|
// Direct source read with native filtering and pagination.
|
|
242
240
|
read: async (ctx) => {
|
|
243
241
|
const page = await queryIssues({
|
|
@@ -258,7 +256,8 @@ A feed's `eventKinds` are also the **default Automation trigger catalog**. The f
|
|
|
258
256
|
|
|
259
257
|
The worker calls the selected feed's `sync` handler for scheduled,
|
|
260
258
|
webhook-triggered, and manually triggered sync runs. It receives a `SyncContext`
|
|
261
|
-
and returns a `SyncResult
|
|
259
|
+
stores events through `ctx.commit`, and returns a `SyncResult` saying whether
|
|
260
|
+
the source is caught up.
|
|
262
261
|
|
|
263
262
|
### SyncContext
|
|
264
263
|
|
|
@@ -271,8 +270,9 @@ interface SyncContext {
|
|
|
271
270
|
credentials: SyncCredentials | null; // OAuth/session credentials; env_keys are in config
|
|
272
271
|
entityIds: number[]; // Linked entity IDs
|
|
273
272
|
sessionState?: Record<string, unknown>; // Browser session state (cookies, tokens)
|
|
274
|
-
|
|
275
|
-
|
|
273
|
+
// Store a page, then move its checkpoint; resolves once both are durable.
|
|
274
|
+
// `null` stores the events and leaves the checkpoint where it is.
|
|
275
|
+
commit: (events: EventEnvelope[], checkpoint: Record<string, unknown> | null) => Promise<void>;
|
|
276
276
|
}
|
|
277
277
|
```
|
|
278
278
|
|
|
@@ -280,8 +280,7 @@ interface SyncContext {
|
|
|
280
280
|
|
|
281
281
|
```typescript
|
|
282
282
|
interface SyncResult {
|
|
283
|
-
|
|
284
|
-
checkpoint: Record<string, unknown> | null; // Updated checkpoint to persist
|
|
283
|
+
status: 'complete' | 'more'; // Caught up, or stopped early with work left (reruns now)
|
|
285
284
|
auth_update?: Record<string, unknown>; // Updated session state (browser cookies, etc.)
|
|
286
285
|
metadata?: {
|
|
287
286
|
items_found?: number;
|
|
@@ -320,7 +319,7 @@ Use checkpoints to implement incremental sync. Common patterns:
|
|
|
320
319
|
- **Timestamp-based**: Store `last_sync_at` and use it as a `since` filter on the next sync (see `github.ts`)
|
|
321
320
|
- **ID-based**: Store a list of seen IDs for deduplication, trimmed to a max size to prevent unbounded growth (see `rss.ts`)
|
|
322
321
|
|
|
323
|
-
|
|
322
|
+
Commit as you go: call `await ctx.commit(events, checkpoint)` once per source page, with the checkpoint that resumes after that page, and await it before fetching the next. A large commit may be transported in chunks, but its checkpoint moves only with the final chunk; if the run is killed or crashes mid-way, replay may supersede already-stored events, and no checkpoint ever covers events that were not stored. A pass that stops early at a budget (pages, rows, time) returns `{ status: 'more' }` so the next run starts immediately; `more` is only honored when the pass moved the checkpoint, so a pass that makes no progress falls back to the schedule instead of looping.
|
|
324
323
|
|
|
325
324
|
## Reading from the Source
|
|
326
325
|
|
|
@@ -527,7 +526,7 @@ Connectors can also be installed manually via `client.connections.installConnect
|
|
|
527
526
|
|
|
528
527
|
1. For fleet workers and embedded-mode hosts (worker + gateway share a host), the gateway sends only `connector_key` in the worker-poll response — both runtimes have the `.ts` source on disk, and the worker compiles locally via the shared pipeline at `@lobu/connector-worker/compile`. For DB-only / device workers without source on disk, the gateway sends `compiled_code` inline.
|
|
529
528
|
2. The bundle is self-contained — the SDK and every pure-JS dependency are inlined — and is evaluated inside a V8 isolate in the worker process. The isolate has no module loader, so a bundle that still `require()`s a Node builtin is rejected before it runs.
|
|
530
|
-
3. Host and guest speak the SDK shapes (`SyncContext` / `ActionContext` / `AuthContext` in, `SyncResult` / `ActionResult` / `AuthResult` out, no envelope) across named host capabilities. Sync
|
|
529
|
+
3. Host and guest speak the SDK shapes (`SyncContext` / `ActionContext` / `AuthContext` in, `SyncResult` / `ActionResult` / `AuthResult` out, no envelope) across named host capabilities. Sync pages and their checkpoints cross the `ctx.commit` capability together.
|
|
531
530
|
4. `IsolateExecutor` defaults to a 10-minute wall clock and a 512 MB heap for every run; interactive auth runs disable the fixed timeout while waiting for the user.
|
|
532
531
|
5. The guest has no filesystem, no ambient environment, and no socket of its own: everything it can reach is a named capability the host granted — `fetch` and, for the DB connectors, a host-dialled TCP socket — plus the explicit values supplied in `job.env`.
|
|
533
532
|
6. Connection credentials and config flow through the typed job context (`ctx.credentials`, `ctx.config`, or auth's `previousCredentials`). The worker API token is never forwarded to connector code. `ctx.credentials.accessToken` is a per-run placeholder the host resolves at the wire (see "`oauth` - OAuth providers" above); `ctx.config` and `previousCredentials` still carry real values.
|
|
@@ -905,12 +905,13 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
905
905
|
if (contentType === 'stargazers') {
|
|
906
906
|
const result = await this.syncStargazers(repo, ctx.checkpoint, token);
|
|
907
907
|
this.stampRepoAttribution(result.events, repo, { attachAutomationSignals });
|
|
908
|
+
// One commit: removed stargazers are only known against the full snapshot.
|
|
909
|
+
await ctx.commit(result.events, {
|
|
910
|
+
last_sync_at: new Date().toISOString(),
|
|
911
|
+
stargazers: result.currentStargazers,
|
|
912
|
+
} as Record<string, unknown>);
|
|
908
913
|
return {
|
|
909
|
-
|
|
910
|
-
checkpoint: {
|
|
911
|
-
last_sync_at: new Date().toISOString(),
|
|
912
|
-
stargazers: result.currentStargazers,
|
|
913
|
-
} as Record<string, unknown>,
|
|
914
|
+
status: 'complete',
|
|
914
915
|
metadata: {
|
|
915
916
|
items_found: result.events.length,
|
|
916
917
|
current_stargazers: result.currentStargazers.length,
|
|
@@ -927,11 +928,11 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
927
928
|
});
|
|
928
929
|
this.stampRepoAttribution(events, repo, { attachAutomationSignals });
|
|
929
930
|
|
|
931
|
+
await ctx.commit(events, {
|
|
932
|
+
last_sync_at: new Date().toISOString(),
|
|
933
|
+
} as Record<string, unknown>);
|
|
930
934
|
return {
|
|
931
|
-
|
|
932
|
-
checkpoint: {
|
|
933
|
-
last_sync_at: new Date().toISOString(),
|
|
934
|
-
} as Record<string, unknown>,
|
|
935
|
+
status: 'complete',
|
|
935
936
|
metadata: {
|
|
936
937
|
items_found: events.length,
|
|
937
938
|
},
|
|
@@ -2090,6 +2091,9 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2090
2091
|
query: params.query,
|
|
2091
2092
|
variables: params.variables ?? {},
|
|
2092
2093
|
},
|
|
2094
|
+
// Every caller sends a query (discussion reads), which is safe to repeat
|
|
2095
|
+
// on a 5xx even though it goes out as POST. A mutation must not set this.
|
|
2096
|
+
idempotent: true,
|
|
2093
2097
|
});
|
|
2094
2098
|
}
|
|
2095
2099
|
|
|
@@ -2101,6 +2105,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2101
2105
|
token: string | null;
|
|
2102
2106
|
body?: Record<string, unknown>;
|
|
2103
2107
|
accept?: string;
|
|
2108
|
+
idempotent?: boolean;
|
|
2104
2109
|
}): Promise<T> {
|
|
2105
2110
|
const headers: Record<string, string> = {
|
|
2106
2111
|
Accept: params.accept ?? 'application/vnd.github+json',
|
|
@@ -2115,6 +2120,7 @@ export default class GitHubConnector extends ConnectorRuntime {
|
|
|
2115
2120
|
method: params.method ?? 'GET',
|
|
2116
2121
|
headers,
|
|
2117
2122
|
body: params.body ? JSON.stringify(params.body) : undefined,
|
|
2123
|
+
idempotent: params.idempotent,
|
|
2118
2124
|
});
|
|
2119
2125
|
}
|
|
2120
2126
|
}
|
|
@@ -457,7 +457,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
457
457
|
const lookbackDays = (ctx.config.lookback_days as number) ?? 30;
|
|
458
458
|
|
|
459
459
|
const checkpoint = (ctx.checkpoint ?? {}) as CalendarCheckpoint;
|
|
460
|
-
const events: EventEnvelope[] = [];
|
|
461
460
|
const durableChanges = ctx.feedKey === 'changes';
|
|
462
461
|
|
|
463
462
|
// A checkpoint minted under a different scope (or by an older version that
|
|
@@ -467,10 +466,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
467
466
|
const scope = JSON.stringify([CHECKPOINT_SCOPE_VERSION, calendarId, lookbackDays]);
|
|
468
467
|
const resumable = checkpoint.scope === scope;
|
|
469
468
|
|
|
470
|
-
/** Traversal reached the last page:
|
|
471
|
-
const finish = (items: EventEnvelope[], syncToken?: string): SyncResult => {
|
|
472
|
-
|
|
473
|
-
|
|
469
|
+
/** Traversal reached the last page: commit it with its cursor and stamp the run. */
|
|
470
|
+
const finish = async (items: EventEnvelope[], syncToken?: string): Promise<SyncResult> => {
|
|
471
|
+
await ctx.commit(items, {
|
|
472
|
+
...this.completedCheckpoint(syncToken),
|
|
473
|
+
scope,
|
|
474
|
+
} as Record<string, unknown>);
|
|
475
|
+
return { status: 'complete' };
|
|
474
476
|
};
|
|
475
477
|
|
|
476
478
|
if (resumable && checkpoint.sync_token) {
|
|
@@ -517,15 +519,14 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
517
519
|
const baseParams = params.toString();
|
|
518
520
|
|
|
519
521
|
/**
|
|
520
|
-
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
522
|
+
* Mid-traversal: the next page token, with no sync token and no
|
|
523
|
+
* `last_sync_at`, so nothing downstream can read it as a completed window.
|
|
524
|
+
* A run that stops here resumes from `token`.
|
|
523
525
|
*/
|
|
524
|
-
const
|
|
525
|
-
|
|
526
|
-
checkpoint: { scope, pending: { params: baseParams, page_token: token } },
|
|
527
|
-
});
|
|
526
|
+
const parked = (token: string) =>
|
|
527
|
+
({ scope, pending: { params: baseParams, page_token: token } }) as Record<string, unknown>;
|
|
528
528
|
|
|
529
|
+
let collected = 0;
|
|
529
530
|
let pageToken = pending?.page_token;
|
|
530
531
|
const seenTokens = new Set<string>();
|
|
531
532
|
for (let page = 0; ; page++) {
|
|
@@ -549,6 +550,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
549
550
|
// A provider page is atomic: `max_results` caps how much a run starts,
|
|
550
551
|
// never how much of a fetched page is stored. Dropping a page's tail
|
|
551
552
|
// would lose those events for good once the cursor moved past them.
|
|
553
|
+
const events: EventEnvelope[] = [];
|
|
552
554
|
for (const calEvent of items) {
|
|
553
555
|
const envelope = durableChanges
|
|
554
556
|
? this.calendarEventToChangeEnvelope(calEvent)
|
|
@@ -570,17 +572,19 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
570
572
|
throw new Error('Google Calendar returned a repeated page token.');
|
|
571
573
|
}
|
|
572
574
|
pageToken = data.nextPageToken;
|
|
575
|
+
await ctx.commit(events, parked(pageToken));
|
|
576
|
+
collected += events.length;
|
|
573
577
|
|
|
574
578
|
// Stop once this run has collected its share, once the provider returns
|
|
575
579
|
// an empty page, or at the hard ceiling — the last of which is the only
|
|
576
580
|
// bound on a window of nothing but filtered-out items, where
|
|
577
|
-
// `
|
|
581
|
+
// `collected` never reaches the cap. The traversal continues next run.
|
|
578
582
|
if (
|
|
579
|
-
|
|
583
|
+
collected >= maxResults ||
|
|
580
584
|
items.length === 0 ||
|
|
581
585
|
page + 1 >= MAX_SYNC_PAGES
|
|
582
586
|
) {
|
|
583
|
-
return
|
|
587
|
+
return { status: 'more' };
|
|
584
588
|
}
|
|
585
589
|
}
|
|
586
590
|
}
|
|
@@ -985,29 +989,15 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
985
989
|
};
|
|
986
990
|
}
|
|
987
991
|
|
|
988
|
-
private
|
|
989
|
-
events: EventEnvelope[],
|
|
990
|
-
syncToken: string | undefined
|
|
991
|
-
): SyncResult {
|
|
992
|
-
// Sort events by occurred_at descending
|
|
993
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
994
|
-
|
|
992
|
+
private completedCheckpoint(syncToken: string | undefined): CalendarCheckpoint {
|
|
995
993
|
// Built fresh rather than spread over the previous checkpoint: the whole
|
|
996
994
|
// object replaces the stored one, so a run that recovered from a rejected
|
|
997
995
|
// token cannot leave that token behind. When the sync produced no new token
|
|
998
996
|
// the key is absent, and the next run correctly starts from a full sync.
|
|
999
|
-
|
|
997
|
+
return {
|
|
1000
998
|
...(syncToken ? { sync_token: syncToken } : {}),
|
|
1001
999
|
last_sync_at: new Date().toISOString(),
|
|
1002
1000
|
};
|
|
1003
|
-
|
|
1004
|
-
return {
|
|
1005
|
-
events,
|
|
1006
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1007
|
-
metadata: {
|
|
1008
|
-
items_found: events.length,
|
|
1009
|
-
},
|
|
1010
|
-
};
|
|
1011
1001
|
}
|
|
1012
1002
|
|
|
1013
1003
|
// Auth-aware client (Bearer + retry/backoff on transient 429/5xx). Built per
|
|
@@ -286,6 +286,27 @@ function buildListQuery(config: DriveConfig): string {
|
|
|
286
286
|
// Connector
|
|
287
287
|
// ---------------------------------------------------------------------------
|
|
288
288
|
|
|
289
|
+
type DriveCommit = (events: EventEnvelope[], checkpoint: DriveCheckpoint) => Promise<void>;
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Checkpoint for a bootstrap with list pages still unread.
|
|
293
|
+
*
|
|
294
|
+
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
295
|
+
* incremental, which is exactly the property that keeps the unread files
|
|
296
|
+
* reachable.
|
|
297
|
+
*/
|
|
298
|
+
function bootstrapCheckpoint(
|
|
299
|
+
pendingPageToken: string,
|
|
300
|
+
listPageToken: string,
|
|
301
|
+
startedAt: string
|
|
302
|
+
): DriveCheckpoint {
|
|
303
|
+
return {
|
|
304
|
+
pending_page_token: pendingPageToken,
|
|
305
|
+
list_page_token: listPageToken,
|
|
306
|
+
last_sync_at: startedAt,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
|
|
289
310
|
export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
290
311
|
Record<string, unknown>,
|
|
291
312
|
DriveConfig
|
|
@@ -493,8 +514,11 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
493
514
|
|
|
494
515
|
// An unfinished bootstrap outranks everything: going incremental with pages
|
|
495
516
|
// still unread would strand every file the traversal never reached.
|
|
517
|
+
const commit = (events: EventEnvelope[], next: DriveCheckpoint) =>
|
|
518
|
+
ctx.commit(events, next as Record<string, unknown>);
|
|
519
|
+
|
|
496
520
|
if (checkpoint.list_page_token && checkpoint.pending_page_token) {
|
|
497
|
-
return this.fullSync(http, config, {
|
|
521
|
+
return this.fullSync(http, config, commit, {
|
|
498
522
|
startToken: checkpoint.pending_page_token,
|
|
499
523
|
listPageToken: checkpoint.list_page_token,
|
|
500
524
|
startedAt: checkpoint.last_sync_at,
|
|
@@ -506,25 +530,15 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
506
530
|
http,
|
|
507
531
|
checkpoint.page_token,
|
|
508
532
|
config,
|
|
533
|
+
commit,
|
|
509
534
|
checkpoint.last_sync_at
|
|
510
535
|
);
|
|
511
|
-
if (result)
|
|
512
|
-
// A partial run must NOT advance `last_sync_at`. The view-churn guard
|
|
513
|
-
// skips content for files modified before that stamp, so stamping now
|
|
514
|
-
// would make the next run treat the unread backlog as already-stored
|
|
515
|
-
// and drop its content on the floor.
|
|
516
|
-
return this.buildResult(
|
|
517
|
-
result.events,
|
|
518
|
-
result.nextPageToken,
|
|
519
|
-
result.partial ? checkpoint.last_sync_at : undefined,
|
|
520
|
-
{ changesPending: result.partial }
|
|
521
|
-
);
|
|
522
|
-
}
|
|
536
|
+
if (result) return result;
|
|
523
537
|
// Token rejected (see isPageTokenRejection). Fall through to ONE full
|
|
524
538
|
// re-list, exactly as the Calendar connector does — never a retry loop.
|
|
525
539
|
}
|
|
526
540
|
|
|
527
|
-
return this.fullSync(http, config);
|
|
541
|
+
return this.fullSync(http, config, commit);
|
|
528
542
|
}
|
|
529
543
|
|
|
530
544
|
/**
|
|
@@ -539,6 +553,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
539
553
|
private async fullSync(
|
|
540
554
|
http: HttpClient,
|
|
541
555
|
config: DriveConfig,
|
|
556
|
+
commit: DriveCommit,
|
|
542
557
|
resume?: { startToken: string; listPageToken: string; startedAt?: string }
|
|
543
558
|
): Promise<SyncResult> {
|
|
544
559
|
// A resumed bootstrap keeps the token minted when the traversal STARTED —
|
|
@@ -555,7 +570,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
555
570
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
556
571
|
const includeContent = config.include_content !== false;
|
|
557
572
|
const q = buildListQuery(config);
|
|
558
|
-
|
|
573
|
+
let collected = 0;
|
|
559
574
|
|
|
560
575
|
// 1000 is Drive's max pageSize, so 200 pages is 200k files — far past any
|
|
561
576
|
// configurable max_results. Defensive bound against a self-referential
|
|
@@ -605,43 +620,53 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
605
620
|
{ maxPages: MAX_PAGES, initialCursor: resume?.listPageToken ?? null }
|
|
606
621
|
);
|
|
607
622
|
|
|
623
|
+
// A Drive `q` has no equivalent on `changes.list` and cannot be re-evaluated
|
|
624
|
+
// client-side against a change record, so the only way to keep honouring it
|
|
625
|
+
// is to keep listing. Withholding the token re-lists next run; promoting it
|
|
626
|
+
// would silently widen the feed to every changed file in the Drive.
|
|
627
|
+
const hasCustomQuery = typeof config.query === 'string' && config.query.trim() !== '';
|
|
628
|
+
|
|
608
629
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
609
630
|
for await (const items of pages) {
|
|
631
|
+
const events: EventEnvelope[] = [];
|
|
610
632
|
for (const file of items) {
|
|
611
633
|
events.push(await this.driveFileToEnvelope(http, file, includeContent, 'upserted'));
|
|
612
634
|
}
|
|
613
|
-
|
|
614
|
-
//
|
|
615
|
-
//
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
635
|
+
collected += events.length;
|
|
636
|
+
// Only Drive withholding a nextPageToken proves the traversal exhausted,
|
|
637
|
+
// so that — not the reason a run stops — is what may promote the change
|
|
638
|
+
// token. Until then each page commits with the bootstrap parked on the
|
|
639
|
+
// next unread page: a page is consumed whole or not at all, so the list
|
|
640
|
+
// token always addresses the first file not yet stored.
|
|
641
|
+
if (!nextListPageToken) {
|
|
642
|
+
await commit(events, {
|
|
643
|
+
...(hasCustomQuery ? {} : { page_token: startToken }),
|
|
644
|
+
last_sync_at: bootstrapStartedAt,
|
|
645
|
+
});
|
|
646
|
+
return {
|
|
647
|
+
status: 'complete',
|
|
648
|
+
metadata: {
|
|
649
|
+
items_found: collected,
|
|
650
|
+
bootstrap_complete: true,
|
|
651
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
652
|
+
},
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
await commit(events, bootstrapCheckpoint(startToken, nextListPageToken, bootstrapStartedAt));
|
|
656
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
632
657
|
}
|
|
633
658
|
|
|
634
|
-
//
|
|
635
|
-
//
|
|
636
|
-
//
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
659
|
+
// Unread pages remain, and WHY this run stopped is irrelevant: the item
|
|
660
|
+
// cap, the clock, and the paginator's own MAX_PAGES ceiling all leave files
|
|
661
|
+
// the change feed will never replay. The bootstrap resumes next run.
|
|
662
|
+
return {
|
|
663
|
+
status: 'more',
|
|
664
|
+
metadata: {
|
|
665
|
+
items_found: collected,
|
|
666
|
+
bootstrap_complete: false,
|
|
667
|
+
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
668
|
+
},
|
|
669
|
+
};
|
|
645
670
|
}
|
|
646
671
|
|
|
647
672
|
/**
|
|
@@ -654,12 +679,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
654
679
|
http: HttpClient,
|
|
655
680
|
pageToken: string,
|
|
656
681
|
config: DriveConfig,
|
|
682
|
+
commit: DriveCommit,
|
|
657
683
|
lastSyncAt?: string
|
|
658
|
-
): Promise<{
|
|
659
|
-
events: EventEnvelope[];
|
|
660
|
-
nextPageToken?: string;
|
|
661
|
-
partial: boolean;
|
|
662
|
-
} | null> {
|
|
684
|
+
): Promise<SyncResult | null> {
|
|
663
685
|
const includeContent = config.include_content !== false;
|
|
664
686
|
const includeTrashed = Boolean(config.include_trashed);
|
|
665
687
|
const folderId =
|
|
@@ -667,7 +689,7 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
667
689
|
? config.folder_id.trim()
|
|
668
690
|
: undefined;
|
|
669
691
|
const maxResults = Math.min(config.max_results ?? 500, 2000);
|
|
670
|
-
|
|
692
|
+
let collected = 0;
|
|
671
693
|
let nextPageToken: string | undefined;
|
|
672
694
|
// The cursor for the page AFTER the one just consumed. A run that stops
|
|
673
695
|
// early persists this instead of dropping back to a full re-list — a
|
|
@@ -718,8 +740,9 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
718
740
|
// start of the next page, so stopping part-way through one would skip the
|
|
719
741
|
// changes still in it.
|
|
720
742
|
const deadline = this.now() + SYNC_TIME_BUDGET_MS;
|
|
721
|
-
let stoppedEarly = false;
|
|
722
743
|
for await (const items of pages) {
|
|
744
|
+
if (rejected) return null;
|
|
745
|
+
const events: EventEnvelope[] = [];
|
|
723
746
|
for (const change of items) {
|
|
724
747
|
const envelope = await this.changeToEnvelope(
|
|
725
748
|
http,
|
|
@@ -731,25 +754,41 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
731
754
|
);
|
|
732
755
|
if (envelope) events.push(envelope);
|
|
733
756
|
}
|
|
734
|
-
|
|
735
|
-
if (
|
|
736
|
-
|
|
737
|
-
|
|
757
|
+
collected += events.length;
|
|
758
|
+
if (!resumeCursor) {
|
|
759
|
+
// The stream is caught up: the page that carried `newStartPageToken`
|
|
760
|
+
// commits it and stamps the run. Rebuilt rather than spread over the
|
|
761
|
+
// previous checkpoint so a run that recovered from a rejected token
|
|
762
|
+
// cannot leave that token behind; without a new token the key is
|
|
763
|
+
// absent and the next run correctly re-lists.
|
|
764
|
+
await commit(events, {
|
|
765
|
+
...(nextPageToken ? { page_token: nextPageToken } : {}),
|
|
766
|
+
last_sync_at: new Date().toISOString(),
|
|
767
|
+
});
|
|
768
|
+
return {
|
|
769
|
+
status: 'complete',
|
|
770
|
+
metadata: { items_found: collected, bootstrap_complete: true },
|
|
771
|
+
};
|
|
738
772
|
}
|
|
773
|
+
// Mid-stream: commit with the cursor of the next page, a resumable
|
|
774
|
+
// position in the change stream. `last_sync_at` does NOT advance — the
|
|
775
|
+
// view-churn guard skips content for files modified before that stamp,
|
|
776
|
+
// so stamping now would make the next run treat the unread backlog as
|
|
777
|
+
// already stored and drop its content on the floor.
|
|
778
|
+
await commit(events, {
|
|
779
|
+
page_token: resumeCursor,
|
|
780
|
+
...(lastSyncAt ? { last_sync_at: lastSyncAt } : {}),
|
|
781
|
+
});
|
|
782
|
+
if (collected >= maxResults || this.now() >= deadline) break;
|
|
739
783
|
}
|
|
740
|
-
|
|
741
784
|
if (rejected) return null;
|
|
742
785
|
|
|
743
|
-
//
|
|
744
|
-
// stream still open
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
return { events, nextPageToken: resumeCursor, partial: true };
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
return { events, nextPageToken, partial: false };
|
|
786
|
+
// Stopped on the cap or the clock, or the paginator ran out of pages
|
|
787
|
+
// (MAX_PAGES) with the stream still open. The committed cursor resumes it.
|
|
788
|
+
return {
|
|
789
|
+
status: 'more',
|
|
790
|
+
metadata: { items_found: collected, bootstrap_complete: true, changes_pending: true },
|
|
791
|
+
};
|
|
753
792
|
}
|
|
754
793
|
|
|
755
794
|
private async fetchStartPageToken(http: HttpClient): Promise<string> {
|
|
@@ -1219,70 +1258,6 @@ export default class GoogleDriveConnector extends ConnectorRuntime<
|
|
|
1219
1258
|
};
|
|
1220
1259
|
}
|
|
1221
1260
|
|
|
1222
|
-
/**
|
|
1223
|
-
* Checkpoint for a bootstrap that stopped on `max_results` with pages left.
|
|
1224
|
-
*
|
|
1225
|
-
* `page_token` is deliberately absent: while it is missing the feed cannot go
|
|
1226
|
-
* incremental, which is exactly the property that keeps the unread files
|
|
1227
|
-
* reachable. `bootstrap_complete: false` is surfaced in metadata so an
|
|
1228
|
-
* operator can see the traversal is still catching up rather than guessing
|
|
1229
|
-
* from a suspiciously round item count.
|
|
1230
|
-
*/
|
|
1231
|
-
private buildBootstrapResult(
|
|
1232
|
-
events: EventEnvelope[],
|
|
1233
|
-
pendingPageToken: string,
|
|
1234
|
-
listPageToken: string,
|
|
1235
|
-
startedAt: string,
|
|
1236
|
-
searchIncomplete = false
|
|
1237
|
-
): SyncResult {
|
|
1238
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1239
|
-
|
|
1240
|
-
const checkpoint: DriveCheckpoint = {
|
|
1241
|
-
pending_page_token: pendingPageToken,
|
|
1242
|
-
list_page_token: listPageToken,
|
|
1243
|
-
last_sync_at: startedAt,
|
|
1244
|
-
};
|
|
1245
|
-
|
|
1246
|
-
return {
|
|
1247
|
-
events,
|
|
1248
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
1249
|
-
metadata: {
|
|
1250
|
-
items_found: events.length,
|
|
1251
|
-
bootstrap_complete: false,
|
|
1252
|
-
...(searchIncomplete ? { search_incomplete: true } : {}),
|
|
1253
|
-
},
|
|
1254
|
-
};
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
private buildResult(
|
|
1258
|
-
events: EventEnvelope[],
|
|
1259
|
-
pageToken: string | undefined,
|
|
1260
|
-
/** Stamp to persist as-is; omit to advance `last_sync_at` to now. */
|
|
1261
|
-
keepLastSyncAt: string | undefined,
|
|
1262
|
-
flags: { changesPending?: boolean; searchIncomplete?: boolean } = {}
|
|
1263
|
-
): SyncResult {
|
|
1264
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
1265
|
-
|
|
1266
|
-
// Rebuilt rather than spread over the previous checkpoint so a run that
|
|
1267
|
-
// recovered from a rejected token cannot leave that token behind. Without a
|
|
1268
|
-
// new token the key is absent and the next run correctly re-lists.
|
|
1269
|
-
const newCheckpoint: DriveCheckpoint = {
|
|
1270
|
-
...(pageToken ? { page_token: pageToken } : {}),
|
|
1271
|
-
last_sync_at: keepLastSyncAt ?? new Date().toISOString(),
|
|
1272
|
-
};
|
|
1273
|
-
|
|
1274
|
-
return {
|
|
1275
|
-
events,
|
|
1276
|
-
checkpoint: newCheckpoint as Record<string, unknown>,
|
|
1277
|
-
metadata: {
|
|
1278
|
-
items_found: events.length,
|
|
1279
|
-
bootstrap_complete: true,
|
|
1280
|
-
...(flags.changesPending ? { changes_pending: true } : {}),
|
|
1281
|
-
...(flags.searchIncomplete ? { search_incomplete: true } : {}),
|
|
1282
|
-
},
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
1261
|
/** Overridable so a test can drive the time budget without sleeping. */
|
|
1287
1262
|
protected now(): number {
|
|
1288
1263
|
return Date.now();
|