@agent-native/core 0.91.1 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/blueprints/action/crud.md +1 -1
- package/blueprints/channel/discord.md +1 -1
- package/blueprints/provider/stripe.md +1 -1
- package/blueprints/sandbox/docker.md +1 -1
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +31 -0
- package/corpus/core/blueprints/action/crud.md +1 -1
- package/corpus/core/blueprints/channel/discord.md +1 -1
- package/corpus/core/blueprints/provider/stripe.md +1 -1
- package/corpus/core/blueprints/sandbox/docker.md +1 -1
- package/corpus/core/docs/content/faq.mdx +17 -0
- package/corpus/core/docs/content/notifications.mdx +12 -6
- package/corpus/core/package.json +6 -5
- package/corpus/core/src/agent/production-agent.ts +140 -46
- package/corpus/core/src/agent/run-manager.ts +23 -2
- package/corpus/core/src/agent/run-store.ts +2 -2
- package/corpus/core/src/agent/tool-search.ts +71 -4
- package/corpus/core/src/cli/add.ts +1 -1
- package/corpus/core/src/cli/create.ts +8 -3
- package/corpus/core/src/cli/index.ts +25 -4
- package/corpus/core/src/cli/upgrade.ts +798 -0
- package/corpus/core/src/client/AssistantChat.tsx +1 -1
- package/corpus/core/src/client/HighlightedCodeBlock.tsx +20 -3
- package/corpus/core/src/client/agent-chat-adapter.ts +31 -6
- package/corpus/core/src/client/chat/message-components.tsx +85 -8
- package/corpus/core/src/client/chat/tool-call-display.tsx +355 -144
- package/corpus/core/src/client/code-agent-chat-adapter.ts +2 -9
- package/corpus/core/src/client/extensions/ExtensionSlot.tsx +7 -0
- package/corpus/core/src/client/settings/SettingsPanel.tsx +533 -389
- package/corpus/core/src/client/settings/SettingsSection.tsx +154 -17
- package/corpus/core/src/client/sse-event-processor.ts +29 -2
- package/corpus/core/src/extensions/actions.ts +113 -5
- package/corpus/core/src/jobs/scheduler.ts +30 -1
- package/corpus/core/src/localization/default-messages.ts +21 -0
- package/corpus/core/src/notifications/channels.ts +88 -25
- package/corpus/core/src/notifications/registry.ts +17 -3
- package/corpus/core/src/notifications/types.ts +4 -1
- package/corpus/core/src/server/agent-chat-plugin.ts +7 -3
- package/corpus/core/src/server/request-context.ts +7 -0
- package/corpus/core/src/styles/agent-conversation.css +1 -1
- package/corpus/core/src/templates/default/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/core/src/templates/default/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/core/src/templates/default/AGENTS.md +10 -3
- package/corpus/core/src/templates/default/app/i18n/ar-SA.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/de-DE.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/en-US.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/es-ES.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/fr-FR.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/hi-IN.ts +20 -0
- package/corpus/core/src/templates/default/app/i18n/ja-JP.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/ko-KR.ts +20 -0
- package/corpus/core/src/templates/default/app/i18n/pt-BR.ts +21 -0
- package/corpus/core/src/templates/default/app/i18n/zh-CN.ts +20 -0
- package/corpus/core/src/templates/default/app/i18n/zh-TW.ts +19 -0
- package/corpus/core/src/templates/default/package.json +4 -3
- package/corpus/core/src/templates/headless/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/core/src/templates/headless/AGENTS.md +9 -3
- package/corpus/core/src/templates/headless/package.json +4 -3
- package/corpus/core/src/templates/workspace-core/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/core/src/templates/workspace-core/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/core/src/templates/workspace-core/package.json +6 -6
- package/corpus/core/src/templates/workspace-root/AGENTS.md +10 -4
- package/corpus/core/src/templates/workspace-root/package.json +3 -2
- package/corpus/core/src/triggers/dispatcher.ts +30 -2
- package/corpus/core/src/vite/client.ts +96 -1
- package/corpus/templates/analytics/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/analytics/.agents/skills/session-replay/SKILL.md +20 -2
- package/corpus/templates/analytics/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/analytics/AGENTS.md +7 -5
- package/corpus/templates/analytics/actions/save-monitor.ts +14 -0
- package/corpus/templates/analytics/app/components/dashboard/DataTable.tsx +41 -6
- package/corpus/templates/analytics/app/components/dashboard/SqlChart.tsx +122 -11
- package/corpus/templates/analytics/app/global.css +26 -10
- package/corpus/templates/analytics/app/i18n/zh-TW.ts +2 -0
- package/corpus/templates/analytics/app/i18n-data.ts +25 -0
- package/corpus/templates/analytics/app/pages/monitoring/MonitoringPage.tsx +8 -2
- package/corpus/templates/analytics/app/pages/monitoring/uptime/MonitorFormPage.tsx +151 -32
- package/corpus/templates/analytics/app/pages/monitoring/uptime/i18n.ts +17 -2
- package/corpus/templates/analytics/app/pages/monitoring/uptime/types.ts +4 -0
- package/corpus/templates/analytics/app/pages/sessions/SessionDetailPage.tsx +461 -286
- package/corpus/templates/analytics/app/pages/sessions/SessionDevToolsPanel.tsx +285 -294
- package/corpus/templates/analytics/changelog/2026-07-08-console-details-expand-inline-under-each-row-and-the-event-t.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-dashboard-tables-keep-their-size-while-loading.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-monitoring-uptime-and-errors-tabs-now-clearly-show-which-view-is-active.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-session-replay-keeps-a-usable-stage-when-dev-tools-is-open-a.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-session-replay-resumes-after-scrubbing-and-expands-console-r.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-session-replay-uses-stock-rrweb-sizing-again-so-recorded.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-session-replays-wait-for-full-load-keep-a-normal-viewport-an.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-sidebar-dashboard-menus-now-open-above-the-dashboard-canvas.md +6 -0
- package/corpus/templates/analytics/changelog/2026-07-08-uptime-alerting-clarifies-in-app-inbox-vs-email-and-lets-you-paste-slack-or-webhook-urls-per-monitor.md +6 -0
- package/corpus/templates/analytics/docs/uptime-monitoring.md +13 -11
- package/corpus/templates/analytics/package.json +2 -2
- package/corpus/templates/analytics/server/db/schema-monitoring.ts +4 -0
- package/corpus/templates/analytics/server/lib/uptime-monitors.ts +60 -12
- package/corpus/templates/analytics/server/plugins/db.ts +10 -0
- package/corpus/templates/assets/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/assets/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/assets/package.json +2 -2
- package/corpus/templates/brain/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/brain/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/brain/package.json +2 -2
- package/corpus/templates/calendar/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/calendar/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/calendar/AGENTS.md +10 -0
- package/corpus/templates/calendar/actions/create-event.ts +5 -1
- package/corpus/templates/calendar/actions/event-action-helpers.ts +41 -0
- package/corpus/templates/calendar/actions/get-attendee-timezones.ts +28 -0
- package/corpus/templates/calendar/actions/set-attendee-timezone.ts +68 -0
- package/corpus/templates/calendar/app/components/calendar/EventAttendeesSection.tsx +150 -11
- package/corpus/templates/calendar/app/components/calendar/EventDetailPanel.tsx +36 -1
- package/corpus/templates/calendar/app/components/calendar/EventDetailPopover.tsx +32 -0
- package/corpus/templates/calendar/app/hooks/use-attendee-timezones.ts +24 -0
- package/corpus/templates/calendar/app/i18n/zh-TW.ts +2 -0
- package/corpus/templates/calendar/app/i18n-data.ts +20 -0
- package/corpus/templates/calendar/app/lib/attendee-local-time.ts +122 -0
- package/corpus/templates/calendar/changelog/2026-07-08-event-details-now-support-extension-widgets-and-you-can-see-.md +6 -0
- package/corpus/templates/calendar/changelog/2026-07-08-when-you-invite-guests-to-a-new-event-you-now-appear-in.md +6 -0
- package/corpus/templates/calendar/package.json +2 -2
- package/corpus/templates/calendar/server/handlers/events.ts +5 -1
- package/corpus/templates/calendar/shared/api.ts +6 -0
- package/corpus/templates/chat/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/chat/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/chat/package.json +2 -2
- package/corpus/templates/clips/.agents/skills/meetings/SKILL.md +6 -2
- package/corpus/templates/clips/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/clips/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/clips/actions/create-meeting.ts +8 -3
- package/corpus/templates/clips/actions/list-meetings.ts +31 -3
- package/corpus/templates/clips/changelog/2026-07-08-after-finishing-a-desktop-recording-reopening-clips-restor.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-clips-now-pops-up-to-take-notes-when-you-join-an-adhoc-zoom-.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-desktop-clip-recordings-are-louder-after-capture-mic-only.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-desktop-meeting-notes-stay-active-from-tray-and-shortcut-controls.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-desktop-meeting-popovers-can-start-notes-and-join-the-meetin.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-meeting-notes-keep-their-recording-state-when-the-desktop.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-meeting-reminder-dropdowns-now-expand-fully-instead-of-being.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-meeting-reminders-now-appear-1-minute-before-start-stay-unti.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-08-the-desktop-meetings-panel-has-room-for-its-drop-shadow-and-.md +6 -0
- package/corpus/templates/clips/chrome-extension/package.json +2 -2
- package/corpus/templates/clips/desktop/package.json +2 -2
- package/corpus/templates/clips/desktop/src/app.tsx +214 -37
- package/corpus/templates/clips/desktop/src/hooks/useMeetingTranscription.ts +30 -6
- package/corpus/templates/clips/desktop/src/lib/meeting-notification-timing.ts +83 -0
- package/corpus/templates/clips/desktop/src/lib/native-recording-warm.ts +21 -0
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +98 -20
- package/corpus/templates/clips/desktop/src/overlays/meeting-notification.tsx +197 -50
- package/corpus/templates/clips/desktop/src/styles.css +150 -53
- package/corpus/templates/clips/desktop/src-tauri/src/adhoc_meetings_watcher.rs +362 -0
- package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +41 -7
- package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +9 -2
- package/corpus/templates/clips/desktop/src-tauri/src/meetings_watcher.rs +89 -11
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +71 -17
- package/corpus/templates/clips/desktop/src-tauri/src/native_speech.rs +36 -1
- package/corpus/templates/clips/desktop/src-tauri/src/notifications.rs +62 -10
- package/corpus/templates/clips/desktop/src-tauri/src/shortcuts.rs +3 -3
- package/corpus/templates/clips/desktop/src-tauri/src/silence_detector.rs +1 -25
- package/corpus/templates/clips/desktop/src-tauri/src/state.rs +5 -0
- package/corpus/templates/clips/desktop/src-tauri/src/tray.rs +18 -6
- package/corpus/templates/clips/desktop/src-tauri/src/util.rs +36 -1
- package/corpus/templates/clips/package.json +2 -2
- package/corpus/templates/content/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/content/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/content/actions/_batch-utils.ts +8 -0
- package/corpus/templates/content/actions/_builder-cms-read-client.ts +7 -5
- package/corpus/templates/content/actions/_content-database-personal-view.ts +80 -0
- package/corpus/templates/content/actions/_database-source-utils.ts +776 -319
- package/corpus/templates/content/actions/_database-utils.ts +6 -1
- package/corpus/templates/content/actions/_property-utils.ts +168 -0
- package/corpus/templates/content/actions/add-content-database-source-field-property.ts +19 -12
- package/corpus/templates/content/actions/bind-content-database-source-field.ts +19 -12
- package/corpus/templates/content/actions/delete-document.ts +9 -13
- package/corpus/templates/content/actions/get-content-database-personal-view.ts +27 -0
- package/corpus/templates/content/actions/refresh-content-database-source.ts +5 -5
- package/corpus/templates/content/actions/update-content-database-personal-view.ts +31 -0
- package/corpus/templates/content/actions/update-content-database-view.ts +3 -0
- package/corpus/templates/content/app/components/editor/DocumentProperties.tsx +1 -1
- package/corpus/templates/content/app/components/editor/body-hydration.ts +35 -11
- package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +2631 -551
- package/corpus/templates/content/app/components/editor/database/filter-sort.ts +220 -18
- package/corpus/templates/content/app/hooks/use-content-database.ts +32 -0
- package/corpus/templates/content/app/hooks/use-documents.ts +8 -2
- package/corpus/templates/content/app/i18n/zh-TW.ts +16 -0
- package/corpus/templates/content/app/i18n-data.ts +154 -0
- package/corpus/templates/content/changelog/2026-07-08-builder-source-sync-now-completes-reliably-on-hosted-databases.md +6 -0
- package/corpus/templates/content/changelog/2026-07-08-database-filters-now-support-notion-style-reset-save-for-eve.md +6 -0
- package/corpus/templates/content/changelog/2026-07-08-date-filters-can-now-target-a-between-range.md +6 -0
- package/corpus/templates/content/changelog/2026-07-08-fixed-builder-database-sync-so-large-sources-continue-past-t.md +6 -0
- package/corpus/templates/content/package.json +2 -2
- package/corpus/templates/content/parity/matrix.md +1 -1
- package/corpus/templates/content/parity/matrix.ts +2 -0
- package/corpus/templates/content/server/plugins/db.ts +19 -0
- package/corpus/templates/content/shared/api.ts +24 -0
- package/corpus/templates/design/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/design/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/design/package.json +2 -2
- package/corpus/templates/dispatch/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/dispatch/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/dispatch/app/i18n/zh-TW.ts +34 -1
- package/corpus/templates/dispatch/app/i18n-data.ts +387 -10
- package/corpus/templates/dispatch/app/routes/automations.tsx +1 -0
- package/corpus/templates/dispatch/changelog/2026-07-08-added-an-automations-page-to-review-pause-and-create.md +6 -0
- package/corpus/templates/dispatch/changelog/2026-07-08-destinations-now-shows-outbound-delivery-queue-health.md +6 -0
- package/corpus/templates/dispatch/package.json +2 -2
- package/corpus/templates/forms/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/forms/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/forms/package.json +2 -2
- package/corpus/templates/macros/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/macros/package.json +2 -2
- package/corpus/templates/mail/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/mail/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/mail/app/hooks/use-emails.ts +56 -44
- package/corpus/templates/mail/app/lib/gmail-mutation-queue.ts +325 -0
- package/corpus/templates/mail/changelog/2026-07-08-rapid-archive-and-mark-read-now-batch-gmail-updates-so-the-i.md +6 -0
- package/corpus/templates/mail/package.json +2 -2
- package/corpus/templates/plan/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/plan/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/plan/package.json +2 -2
- package/corpus/templates/slides/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/corpus/templates/slides/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/slides/app/components/editor/ImageDropPromptPopover.tsx +46 -29
- package/corpus/templates/slides/app/context/DeckContext.tsx +45 -18
- package/corpus/templates/slides/app/i18n/ar-SA.ts +4 -0
- package/corpus/templates/slides/app/i18n/de-DE.ts +4 -0
- package/corpus/templates/slides/app/i18n/en-US.ts +4 -0
- package/corpus/templates/slides/app/i18n/es-ES.ts +4 -0
- package/corpus/templates/slides/app/i18n/fr-FR.ts +4 -0
- package/corpus/templates/slides/app/i18n/hi-IN.ts +4 -0
- package/corpus/templates/slides/app/i18n/ja-JP.ts +4 -0
- package/corpus/templates/slides/app/i18n/ko-KR.ts +4 -0
- package/corpus/templates/slides/app/i18n/pt-BR.ts +4 -0
- package/corpus/templates/slides/app/i18n/zh-CN.ts +4 -0
- package/corpus/templates/slides/app/i18n/zh-TW.ts +4 -0
- package/corpus/templates/slides/app/lib/image-drop-to-agent.ts +115 -0
- package/corpus/templates/slides/app/pages/DeckEditor.tsx +22 -13
- package/corpus/templates/slides/changelog/2026-07-08-dropping-an-image-onto-a-slide-and-sending-it-to-the-agent-w.md +6 -0
- package/corpus/templates/slides/changelog/2026-07-08-undo-works-after-the-agent-edits-a-slide-so-you-can-reverse.md +6 -0
- package/corpus/templates/slides/package.json +2 -2
- package/corpus/templates/videos/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/corpus/templates/voice/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/dist/agent/production-agent.d.ts +5 -1
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +112 -42
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +23 -2
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/agent/run-store.js +2 -2
- package/dist/agent/run-store.js.map +1 -1
- package/dist/agent/tool-search.d.ts +2 -0
- package/dist/agent/tool-search.d.ts.map +1 -1
- package/dist/agent/tool-search.js +60 -4
- package/dist/agent/tool-search.js.map +1 -1
- package/dist/catalog.json +2 -2
- package/dist/cli/add.js +1 -1
- package/dist/cli/add.js.map +1 -1
- package/dist/cli/create.d.ts.map +1 -1
- package/dist/cli/create.js +8 -3
- package/dist/cli/create.js.map +1 -1
- package/dist/cli/index.js +24 -4
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/upgrade.d.ts +100 -0
- package/dist/cli/upgrade.d.ts.map +1 -0
- package/dist/cli/upgrade.js +678 -0
- package/dist/cli/upgrade.js.map +1 -0
- package/dist/client/AssistantChat.js +1 -1
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/HighlightedCodeBlock.d.ts +2 -0
- package/dist/client/HighlightedCodeBlock.d.ts.map +1 -1
- package/dist/client/HighlightedCodeBlock.js +10 -1
- package/dist/client/HighlightedCodeBlock.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +32 -4
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/chat/message-components.d.ts.map +1 -1
- package/dist/client/chat/message-components.js +61 -8
- package/dist/client/chat/message-components.js.map +1 -1
- package/dist/client/chat/tool-call-display.d.ts +12 -0
- package/dist/client/chat/tool-call-display.d.ts.map +1 -1
- package/dist/client/chat/tool-call-display.js +136 -42
- package/dist/client/chat/tool-call-display.js.map +1 -1
- package/dist/client/code-agent-chat-adapter.js +2 -9
- package/dist/client/code-agent-chat-adapter.js.map +1 -1
- package/dist/client/extensions/ExtensionSlot.js +6 -0
- package/dist/client/extensions/ExtensionSlot.js.map +1 -1
- package/dist/client/settings/SettingsPanel.d.ts.map +1 -1
- package/dist/client/settings/SettingsPanel.js +52 -19
- package/dist/client/settings/SettingsPanel.js.map +1 -1
- package/dist/client/settings/SettingsSection.d.ts +19 -4
- package/dist/client/settings/SettingsSection.d.ts.map +1 -1
- package/dist/client/settings/SettingsSection.js +33 -4
- package/dist/client/settings/SettingsSection.js.map +1 -1
- package/dist/client/sse-event-processor.d.ts +9 -0
- package/dist/client/sse-event-processor.d.ts.map +1 -1
- package/dist/client/sse-event-processor.js +20 -2
- package/dist/client/sse-event-processor.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/extensions/actions.d.ts.map +1 -1
- package/dist/extensions/actions.js +85 -5
- package/dist/extensions/actions.js.map +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/jobs/scheduler.js +26 -1
- package/dist/jobs/scheduler.js.map +1 -1
- package/dist/localization/default-messages.d.ts +19 -0
- package/dist/localization/default-messages.d.ts.map +1 -1
- package/dist/localization/default-messages.js +19 -0
- package/dist/localization/default-messages.js.map +1 -1
- package/dist/notifications/channels.d.ts +8 -3
- package/dist/notifications/channels.d.ts.map +1 -1
- package/dist/notifications/channels.js +61 -22
- package/dist/notifications/channels.js.map +1 -1
- package/dist/notifications/registry.d.ts.map +1 -1
- package/dist/notifications/registry.js +14 -3
- package/dist/notifications/registry.js.map +1 -1
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/notifications/types.d.ts +4 -1
- package/dist/notifications/types.d.ts.map +1 -1
- package/dist/notifications/types.js.map +1 -1
- package/dist/observability/routes.d.ts +1 -1
- package/dist/provider-api/corpus-jobs.d.ts +2 -2
- package/dist/resources/handlers.d.ts +3 -3
- package/dist/server/agent-chat-plugin.d.ts +5 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +2 -2
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/request-context.d.ts +7 -0
- package/dist/server/request-context.d.ts.map +1 -1
- package/dist/server/request-context.js.map +1 -1
- package/dist/styles/agent-conversation.css +1 -1
- package/dist/templates/default/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/dist/templates/default/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/dist/templates/default/AGENTS.md +10 -3
- package/dist/templates/default/app/i18n/ar-SA.ts +21 -0
- package/dist/templates/default/app/i18n/de-DE.ts +21 -0
- package/dist/templates/default/app/i18n/en-US.ts +21 -0
- package/dist/templates/default/app/i18n/es-ES.ts +21 -0
- package/dist/templates/default/app/i18n/fr-FR.ts +21 -0
- package/dist/templates/default/app/i18n/hi-IN.ts +20 -0
- package/dist/templates/default/app/i18n/ja-JP.ts +21 -0
- package/dist/templates/default/app/i18n/ko-KR.ts +20 -0
- package/dist/templates/default/app/i18n/pt-BR.ts +21 -0
- package/dist/templates/default/app/i18n/zh-CN.ts +20 -0
- package/dist/templates/default/app/i18n/zh-TW.ts +19 -0
- package/dist/templates/default/package.json +4 -3
- package/dist/templates/headless/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/dist/templates/headless/AGENTS.md +9 -3
- package/dist/templates/headless/package.json +4 -3
- package/dist/templates/workspace-core/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/dist/templates/workspace-core/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/dist/templates/workspace-core/package.json +6 -6
- package/dist/templates/workspace-root/AGENTS.md +10 -4
- package/dist/templates/workspace-root/package.json +3 -2
- package/dist/triggers/dispatcher.js +27 -2
- package/dist/triggers/dispatcher.js.map +1 -1
- package/dist/vite/client.d.ts.map +1 -1
- package/dist/vite/client.js +70 -1
- package/dist/vite/client.js.map +1 -1
- package/docs/content/faq.mdx +17 -0
- package/docs/content/notifications.mdx +12 -6
- package/package.json +7 -6
- package/src/templates/default/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/src/templates/default/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/src/templates/default/AGENTS.md +10 -3
- package/src/templates/default/app/i18n/ar-SA.ts +21 -0
- package/src/templates/default/app/i18n/de-DE.ts +21 -0
- package/src/templates/default/app/i18n/en-US.ts +21 -0
- package/src/templates/default/app/i18n/es-ES.ts +21 -0
- package/src/templates/default/app/i18n/fr-FR.ts +21 -0
- package/src/templates/default/app/i18n/hi-IN.ts +20 -0
- package/src/templates/default/app/i18n/ja-JP.ts +21 -0
- package/src/templates/default/app/i18n/ko-KR.ts +20 -0
- package/src/templates/default/app/i18n/pt-BR.ts +21 -0
- package/src/templates/default/app/i18n/zh-CN.ts +20 -0
- package/src/templates/default/app/i18n/zh-TW.ts +19 -0
- package/src/templates/default/package.json +4 -3
- package/src/templates/headless/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/src/templates/headless/AGENTS.md +9 -3
- package/src/templates/headless/package.json +4 -3
- package/src/templates/workspace-core/.agents/skills/self-modifying-code/SKILL.md +20 -2
- package/src/templates/workspace-core/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
- package/src/templates/workspace-core/package.json +6 -6
- package/src/templates/workspace-root/AGENTS.md +10 -4
- package/src/templates/workspace-root/package.json +3 -2
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { deleteCollabState, releaseDoc } from "@agent-native/core/collab";
|
|
2
|
-
import {
|
|
2
|
+
import { getDialect, type Dialect } from "@agent-native/core/db";
|
|
3
|
+
import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm";
|
|
3
4
|
|
|
4
5
|
import { getDb, schema } from "../server/db/index.js";
|
|
5
6
|
import type {
|
|
@@ -50,6 +51,7 @@ import {
|
|
|
50
51
|
type DocumentPropertyOptionColor,
|
|
51
52
|
} from "../shared/properties.js";
|
|
52
53
|
import { sanitizeNormalizationFormula } from "../shared/properties.js";
|
|
54
|
+
import { chunks } from "./_batch-utils.js";
|
|
53
55
|
import {
|
|
54
56
|
readBuilderCmsContentEntry,
|
|
55
57
|
readBuilderCmsContentEntries,
|
|
@@ -531,12 +533,37 @@ function normalizeBuilderBodyBaselineContent(value: string | null | undefined) {
|
|
|
531
533
|
|
|
532
534
|
const BUILDER_BODY_HYDRATION_BACKGROUND_PRIORITY = 10;
|
|
533
535
|
const BUILDER_BODY_HYDRATION_OPEN_PRIORITY = 0;
|
|
534
|
-
const BUILDER_BODY_HYDRATION_BATCH_LIMIT =
|
|
536
|
+
const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 25;
|
|
537
|
+
const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 6;
|
|
535
538
|
const BUILDER_BODY_HYDRATION_MAX_ATTEMPTS = 5;
|
|
536
539
|
const BUILDER_BODY_HYDRATION_CODEC_VERSION = "readable-native-images-v5";
|
|
537
540
|
const BUILDER_CMS_REFRESH_INITIAL_PAGES = 1;
|
|
538
541
|
const BUILDER_BODY_NOT_AVAILABLE_ERROR = "body not yet available from Builder";
|
|
539
542
|
|
|
543
|
+
export function bulkChunkSizeForColumnCount(
|
|
544
|
+
columnCount: number,
|
|
545
|
+
dialect: Dialect = getDialect(),
|
|
546
|
+
) {
|
|
547
|
+
// D1 rejects statements with more than 100 bound params, so derive every
|
|
548
|
+
// bulk chunk from the statement's column count instead of a fixed row count.
|
|
549
|
+
const budget = dialect === "d1" ? 90 : 900;
|
|
550
|
+
return Math.max(1, Math.floor(budget / Math.max(1, columnCount)));
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
function idChunkSize() {
|
|
554
|
+
return bulkChunkSizeForColumnCount(1);
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
async function processInBatches<T>(
|
|
558
|
+
items: T[],
|
|
559
|
+
concurrency: number,
|
|
560
|
+
worker: (item: T) => Promise<void>,
|
|
561
|
+
) {
|
|
562
|
+
for (const batch of chunks(items, concurrency)) {
|
|
563
|
+
await Promise.all(batch.map((item) => worker(item)));
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
540
567
|
function builderBodyHydrationDelayMs() {
|
|
541
568
|
if (
|
|
542
569
|
process.env.NODE_ENV === "production" ||
|
|
@@ -801,7 +828,7 @@ async function readBuilderEntryWithLiveBodyFromSourceRow(args: {
|
|
|
801
828
|
const liveEntry = await readBuilderCmsContentEntry({
|
|
802
829
|
model: args.sourceTable,
|
|
803
830
|
entryId: args.row.sourceRowId,
|
|
804
|
-
})
|
|
831
|
+
});
|
|
805
832
|
if (!liveEntry || liveEntry.id !== args.row.sourceRowId) return null;
|
|
806
833
|
const entryWithStoredValues = {
|
|
807
834
|
...liveEntry,
|
|
@@ -828,62 +855,134 @@ export async function enqueueBuilderBodyHydration(args: {
|
|
|
828
855
|
now: string;
|
|
829
856
|
priority?: number;
|
|
830
857
|
}) {
|
|
858
|
+
await enqueueBuilderBodyHydrations([args]);
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
type BuilderBodyHydrationEnqueueRequest = {
|
|
862
|
+
sourceId: string;
|
|
863
|
+
ownerEmail: string;
|
|
864
|
+
orgId: string | null;
|
|
865
|
+
databaseItemId: string;
|
|
866
|
+
documentId: string;
|
|
867
|
+
sourceTable: string;
|
|
868
|
+
entry: BuilderCmsSourceEntry;
|
|
869
|
+
now: string;
|
|
870
|
+
priority?: number;
|
|
871
|
+
};
|
|
872
|
+
|
|
873
|
+
async function enqueueBuilderBodyHydrations(
|
|
874
|
+
requests: BuilderBodyHydrationEnqueueRequest[],
|
|
875
|
+
): Promise<ContentDatabaseBodyHydrationQueueRowDb[]> {
|
|
876
|
+
if (requests.length === 0) return [];
|
|
877
|
+
const uniqueRequestsByItemId = new Map<
|
|
878
|
+
string,
|
|
879
|
+
BuilderBodyHydrationEnqueueRequest
|
|
880
|
+
>();
|
|
881
|
+
for (const request of requests) {
|
|
882
|
+
const existing = uniqueRequestsByItemId.get(request.databaseItemId);
|
|
883
|
+
if (!existing) {
|
|
884
|
+
uniqueRequestsByItemId.set(request.databaseItemId, request);
|
|
885
|
+
continue;
|
|
886
|
+
}
|
|
887
|
+
const existingPriority =
|
|
888
|
+
existing.priority ??
|
|
889
|
+
builderBodyHydrationPriorityForRequest({ documentId: null });
|
|
890
|
+
const nextPriority =
|
|
891
|
+
request.priority ??
|
|
892
|
+
builderBodyHydrationPriorityForRequest({ documentId: null });
|
|
893
|
+
uniqueRequestsByItemId.set(request.databaseItemId, {
|
|
894
|
+
...request,
|
|
895
|
+
priority: Math.min(existingPriority, nextPriority),
|
|
896
|
+
});
|
|
897
|
+
}
|
|
898
|
+
const uniqueRequests = Array.from(uniqueRequestsByItemId.values());
|
|
831
899
|
const db = getDb();
|
|
832
|
-
const
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
const
|
|
837
|
-
.
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
900
|
+
const databaseItemIds = Array.from(
|
|
901
|
+
new Set(uniqueRequests.map((request) => request.databaseItemId)),
|
|
902
|
+
);
|
|
903
|
+
const existingRows: ContentDatabaseBodyHydrationQueueRowDb[] = [];
|
|
904
|
+
for (const idChunk of chunks(databaseItemIds, idChunkSize())) {
|
|
905
|
+
existingRows.push(
|
|
906
|
+
...(await db
|
|
907
|
+
.select()
|
|
908
|
+
.from(schema.contentDatabaseBodyHydrationQueue)
|
|
909
|
+
.where(
|
|
910
|
+
inArray(
|
|
911
|
+
schema.contentDatabaseBodyHydrationQueue.databaseItemId,
|
|
912
|
+
idChunk,
|
|
913
|
+
),
|
|
914
|
+
)),
|
|
844
915
|
);
|
|
845
|
-
|
|
846
|
-
const
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
const
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
.
|
|
868
|
-
.
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
createdAt:
|
|
916
|
+
}
|
|
917
|
+
const existingByItemId = new Map(
|
|
918
|
+
existingRows.map((row) => [row.databaseItemId, row]),
|
|
919
|
+
);
|
|
920
|
+
const queueRows: (typeof schema.contentDatabaseBodyHydrationQueue.$inferInsert)[] =
|
|
921
|
+
[];
|
|
922
|
+
for (const request of uniqueRequests) {
|
|
923
|
+
const existing = existingByItemId.get(request.databaseItemId);
|
|
924
|
+
const existingEntry = existing ? parseHydrationEntry(existing) : null;
|
|
925
|
+
const shouldPreserveExistingEntry =
|
|
926
|
+
builderEntryHasBodyContent(existingEntry) &&
|
|
927
|
+
!builderEntryHasBodyContent(request.entry);
|
|
928
|
+
const priority =
|
|
929
|
+
request.priority ??
|
|
930
|
+
builderBodyHydrationPriorityForRequest({ documentId: null });
|
|
931
|
+
queueRows.push({
|
|
932
|
+
id: existing?.id ?? crypto.randomUUID(),
|
|
933
|
+
ownerEmail: request.ownerEmail,
|
|
934
|
+
orgId: request.orgId,
|
|
935
|
+
sourceId: request.sourceId,
|
|
936
|
+
databaseItemId: request.databaseItemId,
|
|
937
|
+
documentId: request.documentId,
|
|
938
|
+
sourceRowId: request.entry.id,
|
|
939
|
+
sourceTable: request.sourceTable,
|
|
940
|
+
sourceEntryJson: shouldPreserveExistingEntry
|
|
941
|
+
? existing!.sourceEntryJson
|
|
942
|
+
: JSON.stringify(request.entry),
|
|
943
|
+
priority: Math.min(existing?.priority ?? priority, priority),
|
|
944
|
+
attempts: existing?.attempts ?? 0,
|
|
945
|
+
lastAttemptedAt: existing?.lastAttemptedAt ?? null,
|
|
946
|
+
lastError: null,
|
|
947
|
+
createdAt: existing?.createdAt ?? request.now,
|
|
948
|
+
updatedAt: request.now,
|
|
877
949
|
});
|
|
878
950
|
}
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
.
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
951
|
+
const upsertedRows: ContentDatabaseBodyHydrationQueueRowDb[] = [];
|
|
952
|
+
for (const chunk of chunks(queueRows, bulkChunkSizeForColumnCount(15))) {
|
|
953
|
+
upsertedRows.push(
|
|
954
|
+
...(await db
|
|
955
|
+
.insert(schema.contentDatabaseBodyHydrationQueue)
|
|
956
|
+
.values(chunk)
|
|
957
|
+
.onConflictDoUpdate({
|
|
958
|
+
target: schema.contentDatabaseBodyHydrationQueue.databaseItemId,
|
|
959
|
+
set: {
|
|
960
|
+
ownerEmail: sql`excluded.owner_email`,
|
|
961
|
+
orgId: sql`excluded.org_id`,
|
|
962
|
+
sourceId: sql`excluded.source_id`,
|
|
963
|
+
documentId: sql`excluded.document_id`,
|
|
964
|
+
sourceRowId: sql`excluded.source_row_id`,
|
|
965
|
+
sourceTable: sql`excluded.source_table`,
|
|
966
|
+
sourceEntryJson: sql`excluded.source_entry_json`,
|
|
967
|
+
priority: sql`excluded.priority`,
|
|
968
|
+
lastError: null,
|
|
969
|
+
updatedAt: sql`excluded.updated_at`,
|
|
970
|
+
},
|
|
971
|
+
})
|
|
972
|
+
.returning()),
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
for (const idChunk of chunks(databaseItemIds, idChunkSize())) {
|
|
976
|
+
await db
|
|
977
|
+
.update(schema.contentDatabaseItems)
|
|
978
|
+
.set({
|
|
979
|
+
bodyHydrationStatus: "pending",
|
|
980
|
+
bodyHydrationError: null,
|
|
981
|
+
updatedAt: uniqueRequests[0]!.now,
|
|
982
|
+
})
|
|
983
|
+
.where(inArray(schema.contentDatabaseItems.id, idChunk));
|
|
984
|
+
}
|
|
985
|
+
return upsertedRows;
|
|
887
986
|
}
|
|
888
987
|
|
|
889
988
|
export async function enqueueBuilderBodyHydrationForItems(args: {
|
|
@@ -896,9 +995,10 @@ export async function enqueueBuilderBodyHydrationForItems(args: {
|
|
|
896
995
|
now: string;
|
|
897
996
|
}) {
|
|
898
997
|
if (!args.builderEntriesByDocumentId?.size) return;
|
|
998
|
+
const requests: BuilderBodyHydrationEnqueueRequest[] = [];
|
|
899
999
|
for (const item of args.items) {
|
|
900
1000
|
const entry = args.builderEntriesByDocumentId.get(item.document.id);
|
|
901
|
-
if (!entry
|
|
1001
|
+
if (!entry) continue;
|
|
902
1002
|
if (
|
|
903
1003
|
item.bodyHydration?.status === "hydrated" &&
|
|
904
1004
|
item.bodyHydration.version === builderBodyHydrationVersion(entry) &&
|
|
@@ -907,7 +1007,7 @@ export async function enqueueBuilderBodyHydrationForItems(args: {
|
|
|
907
1007
|
) {
|
|
908
1008
|
continue;
|
|
909
1009
|
}
|
|
910
|
-
|
|
1010
|
+
requests.push({
|
|
911
1011
|
sourceId: args.sourceId,
|
|
912
1012
|
ownerEmail: args.ownerEmail,
|
|
913
1013
|
orgId: args.orgId,
|
|
@@ -918,9 +1018,11 @@ export async function enqueueBuilderBodyHydrationForItems(args: {
|
|
|
918
1018
|
now: args.now,
|
|
919
1019
|
});
|
|
920
1020
|
}
|
|
1021
|
+
const queuedJobs = await enqueueBuilderBodyHydrations(requests);
|
|
921
1022
|
void processBuilderBodyHydrationQueue({
|
|
922
1023
|
sourceId: args.sourceId,
|
|
923
1024
|
limit: BUILDER_BODY_HYDRATION_BATCH_LIMIT,
|
|
1025
|
+
preloadedJobs: queuedJobs,
|
|
924
1026
|
}).catch((error) => {
|
|
925
1027
|
console.error("Builder body hydration background kick failed", error);
|
|
926
1028
|
});
|
|
@@ -931,11 +1033,25 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
|
|
|
931
1033
|
now: string;
|
|
932
1034
|
}) {
|
|
933
1035
|
const db = getDb();
|
|
1036
|
+
const requests: BuilderBodyHydrationEnqueueRequest[] = [];
|
|
934
1037
|
const rows = await db
|
|
935
1038
|
.select({
|
|
936
|
-
item:
|
|
937
|
-
|
|
938
|
-
|
|
1039
|
+
item: {
|
|
1040
|
+
id: schema.contentDatabaseItems.id,
|
|
1041
|
+
ownerEmail: schema.contentDatabaseItems.ownerEmail,
|
|
1042
|
+
orgId: schema.contentDatabaseItems.orgId,
|
|
1043
|
+
documentId: schema.contentDatabaseItems.documentId,
|
|
1044
|
+
},
|
|
1045
|
+
sourceRow: {
|
|
1046
|
+
sourceRowId: schema.contentDatabaseSourceRows.sourceRowId,
|
|
1047
|
+
sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
|
|
1048
|
+
lastSourceUpdatedAt:
|
|
1049
|
+
schema.contentDatabaseSourceRows.lastSourceUpdatedAt,
|
|
1050
|
+
},
|
|
1051
|
+
document: {
|
|
1052
|
+
title: schema.documents.title,
|
|
1053
|
+
content: schema.documents.content,
|
|
1054
|
+
},
|
|
939
1055
|
})
|
|
940
1056
|
.from(schema.contentDatabaseSourceRows)
|
|
941
1057
|
.innerJoin(
|
|
@@ -949,14 +1065,21 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
|
|
|
949
1065
|
schema.documents,
|
|
950
1066
|
eq(schema.documents.id, schema.contentDatabaseSourceRows.documentId),
|
|
951
1067
|
)
|
|
952
|
-
.where(
|
|
1068
|
+
.where(
|
|
1069
|
+
and(
|
|
1070
|
+
eq(schema.contentDatabaseSourceRows.sourceId, args.source.id),
|
|
1071
|
+
inArray(schema.contentDatabaseItems.bodyHydrationStatus, [
|
|
1072
|
+
"hydrated",
|
|
1073
|
+
"pending",
|
|
1074
|
+
]),
|
|
1075
|
+
or(
|
|
1076
|
+
isNull(schema.documents.content),
|
|
1077
|
+
inArray(schema.documents.content, ["", "<empty-block/>"]),
|
|
1078
|
+
eq(sql<string>`TRIM(${schema.documents.content})`, ""),
|
|
1079
|
+
),
|
|
1080
|
+
),
|
|
1081
|
+
);
|
|
953
1082
|
for (const row of rows) {
|
|
954
|
-
if (
|
|
955
|
-
row.item.bodyHydrationStatus !== "hydrated" &&
|
|
956
|
-
row.item.bodyHydrationStatus !== "pending"
|
|
957
|
-
)
|
|
958
|
-
continue;
|
|
959
|
-
if (!isEffectivelyEmptyDocumentContent(row.document.content)) continue;
|
|
960
1083
|
const entry = builderEntryFromSourceRow({
|
|
961
1084
|
row: row.sourceRow,
|
|
962
1085
|
sourceTable: args.source.sourceTable,
|
|
@@ -965,9 +1088,8 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
|
|
|
965
1088
|
const refreshedEntry = entry
|
|
966
1089
|
? await refreshBuilderBodySourceValuesFromStoredLossless(entry)
|
|
967
1090
|
: null;
|
|
968
|
-
if (!refreshedEntry
|
|
969
|
-
|
|
970
|
-
await enqueueBuilderBodyHydration({
|
|
1091
|
+
if (!refreshedEntry) continue;
|
|
1092
|
+
requests.push({
|
|
971
1093
|
sourceId: args.source.id,
|
|
972
1094
|
ownerEmail: row.item.ownerEmail,
|
|
973
1095
|
orgId: row.item.orgId,
|
|
@@ -978,6 +1100,15 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
|
|
|
978
1100
|
now: args.now,
|
|
979
1101
|
});
|
|
980
1102
|
}
|
|
1103
|
+
const queuedJobs = await enqueueBuilderBodyHydrations(requests);
|
|
1104
|
+
if (queuedJobs.length === 0) return;
|
|
1105
|
+
void processBuilderBodyHydrationQueue({
|
|
1106
|
+
sourceId: args.source.id,
|
|
1107
|
+
limit: BUILDER_BODY_HYDRATION_BATCH_LIMIT,
|
|
1108
|
+
preloadedJobs: queuedJobs,
|
|
1109
|
+
}).catch((error) => {
|
|
1110
|
+
console.error("Builder body hydration repair kick failed", error);
|
|
1111
|
+
});
|
|
981
1112
|
}
|
|
982
1113
|
|
|
983
1114
|
function parseHydrationEntry(
|
|
@@ -990,53 +1121,35 @@ function parseHydrationEntry(
|
|
|
990
1121
|
async function processBuilderBodyHydrationJob(
|
|
991
1122
|
row: ContentDatabaseBodyHydrationQueueRowDb,
|
|
992
1123
|
now: string,
|
|
1124
|
+
preloaded?: {
|
|
1125
|
+
sourceRow?: ContentDatabaseSourceRecordRowDb | null;
|
|
1126
|
+
documentContent?: string | null;
|
|
1127
|
+
},
|
|
993
1128
|
) {
|
|
994
1129
|
const db = getDb();
|
|
995
|
-
const [claimed] = await db
|
|
996
|
-
.update(schema.contentDatabaseBodyHydrationQueue)
|
|
997
|
-
.set({
|
|
998
|
-
attempts: row.attempts + 1,
|
|
999
|
-
lastAttemptedAt: now,
|
|
1000
|
-
lastError: null,
|
|
1001
|
-
updatedAt: now,
|
|
1002
|
-
})
|
|
1003
|
-
.where(
|
|
1004
|
-
and(
|
|
1005
|
-
eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
|
|
1006
|
-
eq(
|
|
1007
|
-
schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
|
|
1008
|
-
row.sourceEntryJson,
|
|
1009
|
-
),
|
|
1010
|
-
),
|
|
1011
|
-
)
|
|
1012
|
-
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1013
|
-
if (!claimed) return;
|
|
1014
|
-
|
|
1015
|
-
await db
|
|
1016
|
-
.update(schema.contentDatabaseItems)
|
|
1017
|
-
.set({
|
|
1018
|
-
bodyHydrationStatus: "hydrating",
|
|
1019
|
-
bodyHydrationAttemptedAt: now,
|
|
1020
|
-
bodyHydrationError: null,
|
|
1021
|
-
updatedAt: now,
|
|
1022
|
-
})
|
|
1023
|
-
.where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
|
|
1024
|
-
|
|
1025
1130
|
const entry = parseHydrationEntry(row);
|
|
1026
1131
|
if (!entry) throw new Error("Builder body hydration entry is missing.");
|
|
1027
1132
|
let activeSourceEntryJson = row.sourceEntryJson;
|
|
1028
1133
|
let entryWithBody = await refreshBuilderBodySourceValuesFromStoredLossless(
|
|
1029
1134
|
await withBuilderBodySourceValues(entry),
|
|
1030
1135
|
);
|
|
1031
|
-
const
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1136
|
+
const sourceRow =
|
|
1137
|
+
preloaded?.sourceRow != null
|
|
1138
|
+
? (preloaded.sourceRow ?? undefined)
|
|
1139
|
+
: (
|
|
1140
|
+
await db
|
|
1141
|
+
.select()
|
|
1142
|
+
.from(schema.contentDatabaseSourceRows)
|
|
1143
|
+
.where(
|
|
1144
|
+
and(
|
|
1145
|
+
eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
|
|
1146
|
+
eq(
|
|
1147
|
+
schema.contentDatabaseSourceRows.databaseItemId,
|
|
1148
|
+
row.databaseItemId,
|
|
1149
|
+
),
|
|
1150
|
+
),
|
|
1151
|
+
)
|
|
1152
|
+
)[0];
|
|
1040
1153
|
const sourceValues =
|
|
1041
1154
|
parseObject<Record<string, DocumentPropertyValue>>(
|
|
1042
1155
|
sourceRow?.sourceValuesJson ?? "{}",
|
|
@@ -1132,20 +1245,40 @@ async function processBuilderBodyHydrationJob(
|
|
|
1132
1245
|
}
|
|
1133
1246
|
}
|
|
1134
1247
|
if (!nextContent.trim()) {
|
|
1135
|
-
const attempts = row.attempts
|
|
1248
|
+
const attempts = row.attempts;
|
|
1136
1249
|
await db.transaction(async (tx) => {
|
|
1137
|
-
|
|
1250
|
+
const queueRowCas = and(
|
|
1251
|
+
eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
|
|
1252
|
+
eq(
|
|
1253
|
+
schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
|
|
1254
|
+
activeSourceEntryJson,
|
|
1255
|
+
),
|
|
1256
|
+
);
|
|
1257
|
+
const markPendingIfReplaced = async () => {
|
|
1258
|
+
const [replacedByNewerJob] = await tx
|
|
1259
|
+
.select({ id: schema.contentDatabaseBodyHydrationQueue.id })
|
|
1260
|
+
.from(schema.contentDatabaseBodyHydrationQueue)
|
|
1261
|
+
.where(eq(schema.contentDatabaseBodyHydrationQueue.id, row.id));
|
|
1262
|
+
if (!replacedByNewerJob) return;
|
|
1138
1263
|
await tx
|
|
1264
|
+
.update(schema.contentDatabaseItems)
|
|
1265
|
+
.set({
|
|
1266
|
+
bodyHydrationStatus: "pending",
|
|
1267
|
+
bodyHydrationAttemptedAt: now,
|
|
1268
|
+
bodyHydrationError: null,
|
|
1269
|
+
updatedAt: now,
|
|
1270
|
+
})
|
|
1271
|
+
.where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
|
|
1272
|
+
};
|
|
1273
|
+
if (builderBodyHydrationAttemptIsTerminal(attempts)) {
|
|
1274
|
+
const [deleted] = await tx
|
|
1139
1275
|
.delete(schema.contentDatabaseBodyHydrationQueue)
|
|
1140
|
-
.where(
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
),
|
|
1147
|
-
),
|
|
1148
|
-
);
|
|
1276
|
+
.where(queueRowCas)
|
|
1277
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1278
|
+
if (!deleted) {
|
|
1279
|
+
await markPendingIfReplaced();
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1149
1282
|
await tx
|
|
1150
1283
|
.update(schema.contentDatabaseItems)
|
|
1151
1284
|
.set({
|
|
@@ -1157,6 +1290,18 @@ async function processBuilderBodyHydrationJob(
|
|
|
1157
1290
|
.where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
|
|
1158
1291
|
return;
|
|
1159
1292
|
}
|
|
1293
|
+
const [stillOwnsQueueRow] = await tx
|
|
1294
|
+
.update(schema.contentDatabaseBodyHydrationQueue)
|
|
1295
|
+
.set({
|
|
1296
|
+
lastError: BUILDER_BODY_NOT_AVAILABLE_ERROR,
|
|
1297
|
+
updatedAt: now,
|
|
1298
|
+
})
|
|
1299
|
+
.where(queueRowCas)
|
|
1300
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1301
|
+
if (!stillOwnsQueueRow) {
|
|
1302
|
+
await markPendingIfReplaced();
|
|
1303
|
+
return;
|
|
1304
|
+
}
|
|
1160
1305
|
await tx
|
|
1161
1306
|
.update(schema.contentDatabaseItems)
|
|
1162
1307
|
.set({
|
|
@@ -1170,13 +1315,18 @@ async function processBuilderBodyHydrationJob(
|
|
|
1170
1315
|
return;
|
|
1171
1316
|
}
|
|
1172
1317
|
}
|
|
1173
|
-
const
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1318
|
+
const documentContent =
|
|
1319
|
+
preloaded?.documentContent != null
|
|
1320
|
+
? preloaded.documentContent
|
|
1321
|
+
: (
|
|
1322
|
+
await db
|
|
1323
|
+
.select({ content: schema.documents.content })
|
|
1324
|
+
.from(schema.documents)
|
|
1325
|
+
.where(eq(schema.documents.id, row.documentId))
|
|
1326
|
+
)[0]?.content;
|
|
1177
1327
|
const previousContent =
|
|
1178
1328
|
stringSourceValue(sourceValues, BUILDER_CMS_BODY_CONTENT_KEY) ?? "";
|
|
1179
|
-
const currentContent =
|
|
1329
|
+
const currentContent = documentContent ?? "";
|
|
1180
1330
|
const shouldWriteBody =
|
|
1181
1331
|
currentContent === "" ||
|
|
1182
1332
|
isEffectivelyEmptyDocumentContent(currentContent) ||
|
|
@@ -1190,6 +1340,40 @@ async function processBuilderBodyHydrationJob(
|
|
|
1190
1340
|
});
|
|
1191
1341
|
let wroteBody = false;
|
|
1192
1342
|
await db.transaction(async (tx) => {
|
|
1343
|
+
const queueRowCas = and(
|
|
1344
|
+
eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
|
|
1345
|
+
eq(
|
|
1346
|
+
schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
|
|
1347
|
+
activeSourceEntryJson,
|
|
1348
|
+
),
|
|
1349
|
+
);
|
|
1350
|
+
const markPendingIfReplaced = async () => {
|
|
1351
|
+
const [replacedByNewerJob] = await tx
|
|
1352
|
+
.select({ id: schema.contentDatabaseBodyHydrationQueue.id })
|
|
1353
|
+
.from(schema.contentDatabaseBodyHydrationQueue)
|
|
1354
|
+
.where(eq(schema.contentDatabaseBodyHydrationQueue.id, row.id));
|
|
1355
|
+
if (!replacedByNewerJob) return;
|
|
1356
|
+
await tx
|
|
1357
|
+
.update(schema.contentDatabaseItems)
|
|
1358
|
+
.set({
|
|
1359
|
+
bodyHydrationStatus: "pending",
|
|
1360
|
+
bodyHydrationAttemptedAt: now,
|
|
1361
|
+
bodyHydrationError: null,
|
|
1362
|
+
updatedAt: now,
|
|
1363
|
+
})
|
|
1364
|
+
.where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
|
|
1365
|
+
};
|
|
1366
|
+
const [stillOwnsQueueRow] = await tx
|
|
1367
|
+
.update(schema.contentDatabaseBodyHydrationQueue)
|
|
1368
|
+
.set({
|
|
1369
|
+
updatedAt: now,
|
|
1370
|
+
})
|
|
1371
|
+
.where(queueRowCas)
|
|
1372
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1373
|
+
if (!stillOwnsQueueRow) {
|
|
1374
|
+
await markPendingIfReplaced();
|
|
1375
|
+
return;
|
|
1376
|
+
}
|
|
1193
1377
|
if (shouldWriteBody) {
|
|
1194
1378
|
const contentCas =
|
|
1195
1379
|
isEffectivelyEmptyDocumentContent(currentContent) &&
|
|
@@ -1215,15 +1399,7 @@ async function processBuilderBodyHydrationJob(
|
|
|
1215
1399
|
"Skipped Builder body hydration because the document changed during sync.",
|
|
1216
1400
|
updatedAt: now,
|
|
1217
1401
|
})
|
|
1218
|
-
.where(
|
|
1219
|
-
and(
|
|
1220
|
-
eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
|
|
1221
|
-
eq(
|
|
1222
|
-
schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
|
|
1223
|
-
activeSourceEntryJson,
|
|
1224
|
-
),
|
|
1225
|
-
),
|
|
1226
|
-
)
|
|
1402
|
+
.where(queueRowCas)
|
|
1227
1403
|
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1228
1404
|
if (stillQueued) {
|
|
1229
1405
|
await tx
|
|
@@ -1239,38 +1415,55 @@ async function processBuilderBodyHydrationJob(
|
|
|
1239
1415
|
}
|
|
1240
1416
|
return;
|
|
1241
1417
|
}
|
|
1242
|
-
const
|
|
1243
|
-
|
|
1244
|
-
.
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
.
|
|
1254
|
-
|
|
1255
|
-
|
|
1418
|
+
const sourceRowWhere = and(
|
|
1419
|
+
eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
|
|
1420
|
+
eq(schema.contentDatabaseSourceRows.databaseItemId, row.databaseItemId),
|
|
1421
|
+
sourceRow
|
|
1422
|
+
? eq(
|
|
1423
|
+
schema.contentDatabaseSourceRows.sourceValuesJson,
|
|
1424
|
+
sourceRow.sourceValuesJson,
|
|
1425
|
+
)
|
|
1426
|
+
: isNull(schema.contentDatabaseSourceRows.id),
|
|
1427
|
+
);
|
|
1428
|
+
const [updatedSourceRow] = await tx
|
|
1429
|
+
.update(schema.contentDatabaseSourceRows)
|
|
1430
|
+
.set({
|
|
1431
|
+
sourceValuesJson: JSON.stringify(nextValues),
|
|
1432
|
+
lastSyncedAt: now,
|
|
1433
|
+
lastSourceUpdatedAt: entryWithBody.updatedAt ?? now,
|
|
1434
|
+
updatedAt: now,
|
|
1435
|
+
})
|
|
1436
|
+
.where(sourceRowWhere)
|
|
1437
|
+
.returning({ id: schema.contentDatabaseSourceRows.id });
|
|
1438
|
+
if (!updatedSourceRow) {
|
|
1439
|
+
const [stillQueued] = await tx
|
|
1440
|
+
.update(schema.contentDatabaseBodyHydrationQueue)
|
|
1441
|
+
.set({
|
|
1442
|
+
lastError:
|
|
1443
|
+
"Skipped Builder body hydration because the source row changed during sync.",
|
|
1444
|
+
updatedAt: now,
|
|
1445
|
+
})
|
|
1446
|
+
.where(queueRowCas)
|
|
1447
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1448
|
+
if (stillQueued) {
|
|
1256
1449
|
await tx
|
|
1257
|
-
.update(schema.
|
|
1450
|
+
.update(schema.contentDatabaseItems)
|
|
1258
1451
|
.set({
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1452
|
+
bodyHydrationStatus: "pending",
|
|
1453
|
+
bodyHydrationAttemptedAt: now,
|
|
1454
|
+
bodyHydrationError:
|
|
1455
|
+
"Skipped Builder body hydration because the source row changed during sync.",
|
|
1262
1456
|
updatedAt: now,
|
|
1263
1457
|
})
|
|
1264
|
-
.where(
|
|
1265
|
-
and(
|
|
1266
|
-
eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
|
|
1267
|
-
eq(
|
|
1268
|
-
schema.contentDatabaseSourceRows.databaseItemId,
|
|
1269
|
-
row.databaseItemId,
|
|
1270
|
-
),
|
|
1271
|
-
),
|
|
1272
|
-
);
|
|
1458
|
+
.where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
|
|
1273
1459
|
}
|
|
1460
|
+
return;
|
|
1461
|
+
}
|
|
1462
|
+
const [deleted] = await tx
|
|
1463
|
+
.delete(schema.contentDatabaseBodyHydrationQueue)
|
|
1464
|
+
.where(queueRowCas)
|
|
1465
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1466
|
+
if (!deleted) {
|
|
1274
1467
|
// Our delete matched nothing: either a NEWER job version replaced this
|
|
1275
1468
|
// row (same id, different sourceEntryJson — mark pending so the newer
|
|
1276
1469
|
// job's processor owns it), or a concurrent processor completed and
|
|
@@ -1293,23 +1486,6 @@ async function processBuilderBodyHydrationJob(
|
|
|
1293
1486
|
}
|
|
1294
1487
|
return;
|
|
1295
1488
|
}
|
|
1296
|
-
await tx
|
|
1297
|
-
.update(schema.contentDatabaseSourceRows)
|
|
1298
|
-
.set({
|
|
1299
|
-
sourceValuesJson: JSON.stringify(nextValues),
|
|
1300
|
-
lastSyncedAt: now,
|
|
1301
|
-
lastSourceUpdatedAt: entryWithBody.updatedAt ?? now,
|
|
1302
|
-
updatedAt: now,
|
|
1303
|
-
})
|
|
1304
|
-
.where(
|
|
1305
|
-
and(
|
|
1306
|
-
eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
|
|
1307
|
-
eq(
|
|
1308
|
-
schema.contentDatabaseSourceRows.databaseItemId,
|
|
1309
|
-
row.databaseItemId,
|
|
1310
|
-
),
|
|
1311
|
-
),
|
|
1312
|
-
);
|
|
1313
1489
|
await tx
|
|
1314
1490
|
.update(schema.contentDatabaseItems)
|
|
1315
1491
|
.set({
|
|
@@ -1408,6 +1584,7 @@ export async function processBuilderBodyHydrationQueue(args: {
|
|
|
1408
1584
|
sourceId: string;
|
|
1409
1585
|
documentId?: string | null;
|
|
1410
1586
|
limit?: number | null;
|
|
1587
|
+
preloadedJobs?: ContentDatabaseBodyHydrationQueueRowDb[];
|
|
1411
1588
|
}) {
|
|
1412
1589
|
const db = getDb();
|
|
1413
1590
|
const limit = normalizeHydrationLimit(args.limit);
|
|
@@ -1434,69 +1611,229 @@ export async function processBuilderBodyHydrationQueue(args: {
|
|
|
1434
1611
|
),
|
|
1435
1612
|
);
|
|
1436
1613
|
}
|
|
1437
|
-
const
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1614
|
+
const persistedJobs = async (queryLimit: number) =>
|
|
1615
|
+
db
|
|
1616
|
+
.select()
|
|
1617
|
+
.from(schema.contentDatabaseBodyHydrationQueue)
|
|
1618
|
+
.where(
|
|
1619
|
+
args.documentId
|
|
1620
|
+
? and(
|
|
1621
|
+
eq(
|
|
1622
|
+
schema.contentDatabaseBodyHydrationQueue.sourceId,
|
|
1623
|
+
args.sourceId,
|
|
1624
|
+
),
|
|
1625
|
+
eq(
|
|
1626
|
+
schema.contentDatabaseBodyHydrationQueue.documentId,
|
|
1627
|
+
args.documentId,
|
|
1628
|
+
),
|
|
1629
|
+
)
|
|
1630
|
+
: eq(
|
|
1444
1631
|
schema.contentDatabaseBodyHydrationQueue.sourceId,
|
|
1445
1632
|
args.sourceId,
|
|
1446
1633
|
),
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1634
|
+
)
|
|
1635
|
+
.orderBy(
|
|
1636
|
+
asc(schema.contentDatabaseBodyHydrationQueue.priority),
|
|
1637
|
+
asc(schema.contentDatabaseBodyHydrationQueue.createdAt),
|
|
1638
|
+
)
|
|
1639
|
+
.limit(queryLimit);
|
|
1640
|
+
const jobs = await (args.preloadedJobs?.length && !args.documentId
|
|
1641
|
+
? (() => {
|
|
1642
|
+
const preloadedJobs = sortBuilderBodyHydrationQueueForProcessing(
|
|
1643
|
+
args.preloadedJobs!.filter((job) => job.sourceId === args.sourceId),
|
|
1644
|
+
).slice(0, limit);
|
|
1645
|
+
return persistedJobs(limit + preloadedJobs.length).then((rows) => {
|
|
1646
|
+
const preloadedIds = new Set(preloadedJobs.map((job) => job.id));
|
|
1647
|
+
return sortBuilderBodyHydrationQueueForProcessing([
|
|
1648
|
+
...preloadedJobs,
|
|
1649
|
+
...rows.filter((row) => !preloadedIds.has(row.id)),
|
|
1650
|
+
]).slice(0, limit);
|
|
1651
|
+
});
|
|
1652
|
+
})()
|
|
1653
|
+
: persistedJobs(limit));
|
|
1459
1654
|
|
|
1460
1655
|
let succeeded = 0;
|
|
1461
1656
|
let failed = 0;
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
bodyHydrationStatus: "error",
|
|
1477
|
-
bodyHydrationAttemptedAt: attemptNow,
|
|
1478
|
-
bodyHydrationError: message,
|
|
1479
|
-
updatedAt: attemptNow,
|
|
1480
|
-
})
|
|
1481
|
-
.where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
|
|
1482
|
-
if (builderBodyHydrationAttemptIsTerminal(attempts)) {
|
|
1483
|
-
await db
|
|
1484
|
-
.delete(schema.contentDatabaseBodyHydrationQueue)
|
|
1485
|
-
.where(eq(schema.contentDatabaseBodyHydrationQueue.id, job.id));
|
|
1486
|
-
continue;
|
|
1487
|
-
}
|
|
1488
|
-
await db
|
|
1657
|
+
const claimedJobs: ContentDatabaseBodyHydrationQueueRowDb[] = [];
|
|
1658
|
+
for (const jobChunk of chunks(jobs, bulkChunkSizeForColumnCount(2))) {
|
|
1659
|
+
const claimFilters = jobChunk.map((job) =>
|
|
1660
|
+
and(
|
|
1661
|
+
eq(schema.contentDatabaseBodyHydrationQueue.id, job.id),
|
|
1662
|
+
eq(
|
|
1663
|
+
schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
|
|
1664
|
+
job.sourceEntryJson,
|
|
1665
|
+
),
|
|
1666
|
+
),
|
|
1667
|
+
);
|
|
1668
|
+
if (claimFilters.length === 0) continue;
|
|
1669
|
+
claimedJobs.push(
|
|
1670
|
+
...(await db
|
|
1489
1671
|
.update(schema.contentDatabaseBodyHydrationQueue)
|
|
1490
1672
|
.set({
|
|
1491
|
-
attempts
|
|
1492
|
-
lastAttemptedAt:
|
|
1493
|
-
lastError:
|
|
1494
|
-
|
|
1495
|
-
updatedAt: attemptNow,
|
|
1673
|
+
attempts: sql`${schema.contentDatabaseBodyHydrationQueue.attempts} + 1`,
|
|
1674
|
+
lastAttemptedAt: now,
|
|
1675
|
+
lastError: null,
|
|
1676
|
+
updatedAt: now,
|
|
1496
1677
|
})
|
|
1497
|
-
.where(
|
|
1498
|
-
|
|
1678
|
+
.where(or(...claimFilters))
|
|
1679
|
+
.returning()),
|
|
1680
|
+
);
|
|
1681
|
+
}
|
|
1682
|
+
for (const idChunk of chunks(
|
|
1683
|
+
claimedJobs.map((job) => job.databaseItemId),
|
|
1684
|
+
idChunkSize(),
|
|
1685
|
+
)) {
|
|
1686
|
+
await db
|
|
1687
|
+
.update(schema.contentDatabaseItems)
|
|
1688
|
+
.set({
|
|
1689
|
+
bodyHydrationStatus: "hydrating",
|
|
1690
|
+
bodyHydrationAttemptedAt: now,
|
|
1691
|
+
bodyHydrationError: null,
|
|
1692
|
+
updatedAt: now,
|
|
1693
|
+
})
|
|
1694
|
+
.where(inArray(schema.contentDatabaseItems.id, idChunk));
|
|
1499
1695
|
}
|
|
1696
|
+
const sourceRows =
|
|
1697
|
+
claimedJobs.length > 0
|
|
1698
|
+
? (
|
|
1699
|
+
await Promise.all(
|
|
1700
|
+
chunks(
|
|
1701
|
+
claimedJobs.map((job) => job.databaseItemId),
|
|
1702
|
+
idChunkSize(),
|
|
1703
|
+
).map((idChunk) =>
|
|
1704
|
+
db
|
|
1705
|
+
.select()
|
|
1706
|
+
.from(schema.contentDatabaseSourceRows)
|
|
1707
|
+
.where(
|
|
1708
|
+
and(
|
|
1709
|
+
eq(
|
|
1710
|
+
schema.contentDatabaseSourceRows.sourceId,
|
|
1711
|
+
args.sourceId,
|
|
1712
|
+
),
|
|
1713
|
+
inArray(
|
|
1714
|
+
schema.contentDatabaseSourceRows.databaseItemId,
|
|
1715
|
+
idChunk,
|
|
1716
|
+
),
|
|
1717
|
+
),
|
|
1718
|
+
),
|
|
1719
|
+
),
|
|
1720
|
+
)
|
|
1721
|
+
).flat()
|
|
1722
|
+
: [];
|
|
1723
|
+
const sourceRowsByItemId = new Map(
|
|
1724
|
+
sourceRows.map((row) => [row.databaseItemId, row]),
|
|
1725
|
+
);
|
|
1726
|
+
const documents =
|
|
1727
|
+
claimedJobs.length > 0
|
|
1728
|
+
? (
|
|
1729
|
+
await Promise.all(
|
|
1730
|
+
chunks(
|
|
1731
|
+
Array.from(new Set(claimedJobs.map((job) => job.documentId))),
|
|
1732
|
+
idChunkSize(),
|
|
1733
|
+
).map((idChunk) =>
|
|
1734
|
+
db
|
|
1735
|
+
.select({
|
|
1736
|
+
id: schema.documents.id,
|
|
1737
|
+
content: schema.documents.content,
|
|
1738
|
+
})
|
|
1739
|
+
.from(schema.documents)
|
|
1740
|
+
.where(inArray(schema.documents.id, idChunk)),
|
|
1741
|
+
),
|
|
1742
|
+
)
|
|
1743
|
+
).flat()
|
|
1744
|
+
: [];
|
|
1745
|
+
const documentContentById = new Map(
|
|
1746
|
+
documents.map((document) => [document.id, document.content]),
|
|
1747
|
+
);
|
|
1748
|
+
await processInBatches(
|
|
1749
|
+
claimedJobs,
|
|
1750
|
+
BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY,
|
|
1751
|
+
async (job) => {
|
|
1752
|
+
const attemptNow = new Date().toISOString();
|
|
1753
|
+
try {
|
|
1754
|
+
const delayMs = builderBodyHydrationDelayMs();
|
|
1755
|
+
if (delayMs > 0) await sleep(delayMs);
|
|
1756
|
+
await processBuilderBodyHydrationJob(job, attemptNow, {
|
|
1757
|
+
sourceRow: sourceRowsByItemId.get(job.databaseItemId) ?? null,
|
|
1758
|
+
documentContent: documentContentById.has(job.documentId)
|
|
1759
|
+
? (documentContentById.get(job.documentId) ?? null)
|
|
1760
|
+
: undefined,
|
|
1761
|
+
});
|
|
1762
|
+
succeeded += 1;
|
|
1763
|
+
} catch (error) {
|
|
1764
|
+
failed += 1;
|
|
1765
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1766
|
+
const attempts = job.attempts;
|
|
1767
|
+
const queueRowCas = and(
|
|
1768
|
+
eq(schema.contentDatabaseBodyHydrationQueue.id, job.id),
|
|
1769
|
+
eq(
|
|
1770
|
+
schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
|
|
1771
|
+
job.sourceEntryJson,
|
|
1772
|
+
),
|
|
1773
|
+
);
|
|
1774
|
+
const markPendingIfReplaced = async () => {
|
|
1775
|
+
const [replacedByNewerJob] = await db
|
|
1776
|
+
.select({ id: schema.contentDatabaseBodyHydrationQueue.id })
|
|
1777
|
+
.from(schema.contentDatabaseBodyHydrationQueue)
|
|
1778
|
+
.where(eq(schema.contentDatabaseBodyHydrationQueue.id, job.id));
|
|
1779
|
+
if (!replacedByNewerJob) return;
|
|
1780
|
+
await db
|
|
1781
|
+
.update(schema.contentDatabaseItems)
|
|
1782
|
+
.set({
|
|
1783
|
+
bodyHydrationStatus: "pending",
|
|
1784
|
+
bodyHydrationAttemptedAt: attemptNow,
|
|
1785
|
+
bodyHydrationError: null,
|
|
1786
|
+
updatedAt: attemptNow,
|
|
1787
|
+
})
|
|
1788
|
+
.where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
|
|
1789
|
+
};
|
|
1790
|
+
if (builderBodyHydrationAttemptIsTerminal(attempts)) {
|
|
1791
|
+
const [deleted] = await db
|
|
1792
|
+
.delete(schema.contentDatabaseBodyHydrationQueue)
|
|
1793
|
+
.where(queueRowCas)
|
|
1794
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1795
|
+
if (!deleted) {
|
|
1796
|
+
await markPendingIfReplaced();
|
|
1797
|
+
return;
|
|
1798
|
+
}
|
|
1799
|
+
await db
|
|
1800
|
+
.update(schema.contentDatabaseItems)
|
|
1801
|
+
.set({
|
|
1802
|
+
bodyHydrationStatus: "error",
|
|
1803
|
+
bodyHydrationAttemptedAt: attemptNow,
|
|
1804
|
+
bodyHydrationError: message,
|
|
1805
|
+
updatedAt: attemptNow,
|
|
1806
|
+
})
|
|
1807
|
+
.where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
|
|
1808
|
+
return;
|
|
1809
|
+
}
|
|
1810
|
+
const [updatedQueueRow] = await db
|
|
1811
|
+
.update(schema.contentDatabaseBodyHydrationQueue)
|
|
1812
|
+
.set({
|
|
1813
|
+
attempts,
|
|
1814
|
+
lastAttemptedAt: attemptNow,
|
|
1815
|
+
lastError: message,
|
|
1816
|
+
priority: job.priority + 10,
|
|
1817
|
+
updatedAt: attemptNow,
|
|
1818
|
+
})
|
|
1819
|
+
.where(queueRowCas)
|
|
1820
|
+
.returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
|
|
1821
|
+
if (!updatedQueueRow) {
|
|
1822
|
+
await markPendingIfReplaced();
|
|
1823
|
+
return;
|
|
1824
|
+
}
|
|
1825
|
+
await db
|
|
1826
|
+
.update(schema.contentDatabaseItems)
|
|
1827
|
+
.set({
|
|
1828
|
+
bodyHydrationStatus: "error",
|
|
1829
|
+
bodyHydrationAttemptedAt: attemptNow,
|
|
1830
|
+
bodyHydrationError: message,
|
|
1831
|
+
updatedAt: attemptNow,
|
|
1832
|
+
})
|
|
1833
|
+
.where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
|
|
1834
|
+
}
|
|
1835
|
+
},
|
|
1836
|
+
);
|
|
1500
1837
|
const [remaining] = await db
|
|
1501
1838
|
.select({ count: sql<number>`COUNT(*)` })
|
|
1502
1839
|
.from(schema.contentDatabaseBodyHydrationQueue)
|
|
@@ -1505,7 +1842,7 @@ export async function processBuilderBodyHydrationQueue(args: {
|
|
|
1505
1842
|
);
|
|
1506
1843
|
return {
|
|
1507
1844
|
sourceId: args.sourceId,
|
|
1508
|
-
processed:
|
|
1845
|
+
processed: claimedJobs.length,
|
|
1509
1846
|
succeeded,
|
|
1510
1847
|
failed,
|
|
1511
1848
|
remaining: Number(remaining?.count ?? 0),
|
|
@@ -3160,7 +3497,7 @@ function builderSourceValuesWithPreservedBodyBaseline(args: {
|
|
|
3160
3497
|
return next;
|
|
3161
3498
|
}
|
|
3162
3499
|
|
|
3163
|
-
async function materializeSourceFieldPropertyValues(args: {
|
|
3500
|
+
export async function materializeSourceFieldPropertyValues(args: {
|
|
3164
3501
|
database: ContentDatabaseRow;
|
|
3165
3502
|
sourceId: string;
|
|
3166
3503
|
fields: ContentDatabaseSourceFieldRowDb[];
|
|
@@ -3184,25 +3521,104 @@ async function materializeSourceFieldPropertyValues(args: {
|
|
|
3184
3521
|
const definitionById = new Map(
|
|
3185
3522
|
definitions.map((definition) => [definition.id, definition]),
|
|
3186
3523
|
);
|
|
3187
|
-
const
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3524
|
+
const scopedRows: Array<{
|
|
3525
|
+
documentId: string;
|
|
3526
|
+
sourceValuesJson: string;
|
|
3527
|
+
}> = [];
|
|
3528
|
+
if (documentIdSet) {
|
|
3529
|
+
for (const idChunk of chunks(Array.from(documentIdSet), idChunkSize())) {
|
|
3530
|
+
scopedRows.push(
|
|
3531
|
+
...(await db
|
|
3532
|
+
.select({
|
|
3533
|
+
documentId: schema.contentDatabaseSourceRows.documentId,
|
|
3534
|
+
sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
|
|
3535
|
+
})
|
|
3536
|
+
.from(schema.contentDatabaseSourceRows)
|
|
3537
|
+
.where(
|
|
3538
|
+
and(
|
|
3539
|
+
eq(schema.contentDatabaseSourceRows.sourceId, args.sourceId),
|
|
3540
|
+
inArray(schema.contentDatabaseSourceRows.documentId, idChunk),
|
|
3541
|
+
),
|
|
3542
|
+
)
|
|
3543
|
+
.orderBy(
|
|
3544
|
+
asc(schema.contentDatabaseSourceRows.updatedAt),
|
|
3545
|
+
asc(schema.contentDatabaseSourceRows.createdAt),
|
|
3546
|
+
asc(schema.contentDatabaseSourceRows.id),
|
|
3547
|
+
)),
|
|
3548
|
+
);
|
|
3549
|
+
}
|
|
3550
|
+
} else {
|
|
3551
|
+
scopedRows.push(
|
|
3552
|
+
...(await db
|
|
3553
|
+
.select({
|
|
3554
|
+
documentId: schema.contentDatabaseSourceRows.documentId,
|
|
3555
|
+
sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
|
|
3556
|
+
})
|
|
3557
|
+
.from(schema.contentDatabaseSourceRows)
|
|
3558
|
+
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.sourceId))
|
|
3559
|
+
.orderBy(
|
|
3560
|
+
asc(schema.contentDatabaseSourceRows.updatedAt),
|
|
3561
|
+
asc(schema.contentDatabaseSourceRows.createdAt),
|
|
3562
|
+
asc(schema.contentDatabaseSourceRows.id),
|
|
3563
|
+
)),
|
|
3564
|
+
);
|
|
3565
|
+
}
|
|
3194
3566
|
if (scopedRows.length === 0) return;
|
|
3195
3567
|
|
|
3196
|
-
const existingValues
|
|
3197
|
-
.
|
|
3198
|
-
|
|
3199
|
-
|
|
3568
|
+
const existingValues: Array<
|
|
3569
|
+
typeof schema.documentPropertyValues.$inferSelect
|
|
3570
|
+
> = [];
|
|
3571
|
+
const scopedExistingValueChunkSize = documentIdSet
|
|
3572
|
+
? bulkChunkSizeForColumnCount(2)
|
|
3573
|
+
: idChunkSize();
|
|
3574
|
+
for (const propertyIdChunk of chunks(
|
|
3575
|
+
propertyIds,
|
|
3576
|
+
scopedExistingValueChunkSize,
|
|
3577
|
+
)) {
|
|
3578
|
+
if (documentIdSet) {
|
|
3579
|
+
for (const documentIdChunk of chunks(
|
|
3580
|
+
Array.from(documentIdSet),
|
|
3581
|
+
scopedExistingValueChunkSize,
|
|
3582
|
+
)) {
|
|
3583
|
+
existingValues.push(
|
|
3584
|
+
...(await db
|
|
3585
|
+
.select()
|
|
3586
|
+
.from(schema.documentPropertyValues)
|
|
3587
|
+
.where(
|
|
3588
|
+
and(
|
|
3589
|
+
inArray(
|
|
3590
|
+
schema.documentPropertyValues.propertyId,
|
|
3591
|
+
propertyIdChunk,
|
|
3592
|
+
),
|
|
3593
|
+
inArray(
|
|
3594
|
+
schema.documentPropertyValues.documentId,
|
|
3595
|
+
documentIdChunk,
|
|
3596
|
+
),
|
|
3597
|
+
),
|
|
3598
|
+
)),
|
|
3599
|
+
);
|
|
3600
|
+
}
|
|
3601
|
+
} else {
|
|
3602
|
+
existingValues.push(
|
|
3603
|
+
...(await db
|
|
3604
|
+
.select()
|
|
3605
|
+
.from(schema.documentPropertyValues)
|
|
3606
|
+
.where(
|
|
3607
|
+
inArray(schema.documentPropertyValues.propertyId, propertyIdChunk),
|
|
3608
|
+
)),
|
|
3609
|
+
);
|
|
3610
|
+
}
|
|
3611
|
+
}
|
|
3200
3612
|
const existingByDocumentAndProperty = new Map(
|
|
3201
3613
|
existingValues.map((value) => [
|
|
3202
3614
|
`${value.documentId}\0${value.propertyId}`,
|
|
3203
3615
|
value,
|
|
3204
3616
|
]),
|
|
3205
3617
|
);
|
|
3618
|
+
const upsertsByDocumentAndProperty = new Map<
|
|
3619
|
+
string,
|
|
3620
|
+
typeof schema.documentPropertyValues.$inferInsert
|
|
3621
|
+
>();
|
|
3206
3622
|
|
|
3207
3623
|
for (const row of scopedRows) {
|
|
3208
3624
|
const sourceValues =
|
|
@@ -3222,27 +3638,34 @@ async function materializeSourceFieldPropertyValues(args: {
|
|
|
3222
3638
|
const valueJson = serializePropertyValue(normalized);
|
|
3223
3639
|
const key = `${row.documentId}\0${propertyId}`;
|
|
3224
3640
|
const existing = existingByDocumentAndProperty.get(key);
|
|
3225
|
-
if (existing)
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
documentId: row.documentId,
|
|
3236
|
-
propertyId,
|
|
3237
|
-
valueJson,
|
|
3238
|
-
createdAt: args.now,
|
|
3239
|
-
updatedAt: args.now,
|
|
3240
|
-
};
|
|
3241
|
-
await db.insert(schema.documentPropertyValues).values(value);
|
|
3242
|
-
existingByDocumentAndProperty.set(key, value);
|
|
3243
|
-
}
|
|
3641
|
+
if (existing?.valueJson === valueJson) continue;
|
|
3642
|
+
upsertsByDocumentAndProperty.set(key, {
|
|
3643
|
+
id: existing?.id ?? nanoid(),
|
|
3644
|
+
ownerEmail: existing?.ownerEmail ?? args.database.ownerEmail,
|
|
3645
|
+
documentId: row.documentId,
|
|
3646
|
+
propertyId,
|
|
3647
|
+
valueJson,
|
|
3648
|
+
createdAt: existing?.createdAt ?? args.now,
|
|
3649
|
+
updatedAt: args.now,
|
|
3650
|
+
});
|
|
3244
3651
|
}
|
|
3245
3652
|
}
|
|
3653
|
+
for (const chunk of chunks(
|
|
3654
|
+
Array.from(upsertsByDocumentAndProperty.values()),
|
|
3655
|
+
bulkChunkSizeForColumnCount(7),
|
|
3656
|
+
)) {
|
|
3657
|
+
if (chunk.length === 0) continue;
|
|
3658
|
+
await db
|
|
3659
|
+
.insert(schema.documentPropertyValues)
|
|
3660
|
+
.values(chunk)
|
|
3661
|
+
.onConflictDoUpdate({
|
|
3662
|
+
target: schema.documentPropertyValues.id,
|
|
3663
|
+
set: {
|
|
3664
|
+
valueJson: sql`excluded.value_json`,
|
|
3665
|
+
updatedAt: sql`excluded.updated_at`,
|
|
3666
|
+
},
|
|
3667
|
+
});
|
|
3668
|
+
}
|
|
3246
3669
|
}
|
|
3247
3670
|
|
|
3248
3671
|
function openChangeSetKey(row: ContentDatabaseSourceChangeSetRowDb) {
|
|
@@ -3643,7 +4066,8 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
|
|
|
3643
4066
|
|
|
3644
4067
|
let nextDocPosition = (maxDocPos?.max ?? -1) + 1;
|
|
3645
4068
|
let nextItemPosition = (maxItemPos?.max ?? -1) + 1;
|
|
3646
|
-
|
|
4069
|
+
const documentRows: (typeof schema.documents.$inferInsert)[] = [];
|
|
4070
|
+
const itemRows: (typeof schema.contentDatabaseItems.$inferInsert)[] = [];
|
|
3647
4071
|
|
|
3648
4072
|
for (const entry of args.entries) {
|
|
3649
4073
|
if (
|
|
@@ -3663,7 +4087,7 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
|
|
|
3663
4087
|
|
|
3664
4088
|
const documentId = nanoid();
|
|
3665
4089
|
const itemId = nanoid();
|
|
3666
|
-
|
|
4090
|
+
documentRows.push({
|
|
3667
4091
|
id: documentId,
|
|
3668
4092
|
ownerEmail: args.database.ownerEmail,
|
|
3669
4093
|
orgId: args.database.orgId,
|
|
@@ -3678,23 +4102,30 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
|
|
|
3678
4102
|
createdAt: args.now,
|
|
3679
4103
|
updatedAt: args.now,
|
|
3680
4104
|
});
|
|
3681
|
-
|
|
4105
|
+
itemRows.push({
|
|
3682
4106
|
id: itemId,
|
|
3683
4107
|
ownerEmail: args.database.ownerEmail,
|
|
3684
4108
|
orgId: args.database.orgId,
|
|
3685
4109
|
databaseId: args.database.id,
|
|
3686
4110
|
documentId,
|
|
3687
4111
|
position: nextItemPosition++,
|
|
3688
|
-
bodyHydrationStatus:
|
|
4112
|
+
bodyHydrationStatus: "pending",
|
|
3689
4113
|
bodyHydrationError: null,
|
|
3690
4114
|
createdAt: args.now,
|
|
3691
4115
|
updatedAt: args.now,
|
|
3692
4116
|
});
|
|
3693
|
-
imported += 1;
|
|
3694
4117
|
importedEntriesByDocumentId.set(documentId, entry);
|
|
3695
4118
|
}
|
|
4119
|
+
await db.transaction(async (tx) => {
|
|
4120
|
+
for (const chunk of chunks(documentRows, bulkChunkSizeForColumnCount(13))) {
|
|
4121
|
+
await tx.insert(schema.documents).values(chunk);
|
|
4122
|
+
}
|
|
4123
|
+
for (const chunk of chunks(itemRows, bulkChunkSizeForColumnCount(10))) {
|
|
4124
|
+
await tx.insert(schema.contentDatabaseItems).values(chunk);
|
|
4125
|
+
}
|
|
4126
|
+
});
|
|
3696
4127
|
|
|
3697
|
-
return { imported, importedEntriesByDocumentId };
|
|
4128
|
+
return { imported: itemRows.length, importedEntriesByDocumentId };
|
|
3698
4129
|
}
|
|
3699
4130
|
|
|
3700
4131
|
export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
@@ -3765,9 +4196,19 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
3765
4196
|
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
|
|
3766
4197
|
}
|
|
3767
4198
|
}
|
|
3768
|
-
|
|
3769
|
-
|
|
3770
|
-
|
|
4199
|
+
let builderModelFields: BuilderCmsModelFieldSummary[] | undefined;
|
|
4200
|
+
let builderModelFieldsReadFailed = false;
|
|
4201
|
+
try {
|
|
4202
|
+
builderModelFields = await readBuilderCmsModelFields({
|
|
4203
|
+
model: args.source.sourceTable,
|
|
4204
|
+
});
|
|
4205
|
+
} catch (error) {
|
|
4206
|
+
builderModelFieldsReadFailed = true;
|
|
4207
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4208
|
+
console.warn(
|
|
4209
|
+
`[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`,
|
|
4210
|
+
);
|
|
4211
|
+
}
|
|
3771
4212
|
const builderEntriesByDocumentId =
|
|
3772
4213
|
builderRead.state === "live"
|
|
3773
4214
|
? mapBuilderCmsEntriesToLocalItems({
|
|
@@ -3806,36 +4247,39 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
3806
4247
|
);
|
|
3807
4248
|
const hasMore = builderRead.progress?.hasMore === true;
|
|
3808
4249
|
|
|
3809
|
-
|
|
3810
|
-
.
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
4250
|
+
const shouldReseedSourceFields =
|
|
4251
|
+
!builderModelFieldsReadFailed || existingFields.length === 0;
|
|
4252
|
+
if (shouldReseedSourceFields) {
|
|
4253
|
+
await db
|
|
4254
|
+
.delete(schema.contentDatabaseSourceFields)
|
|
4255
|
+
.where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
|
|
4256
|
+
await seedMockSourceFields({
|
|
4257
|
+
sourceId: args.source.id,
|
|
4258
|
+
ownerEmail: args.database.ownerEmail,
|
|
4259
|
+
sourceType: "builder-cms",
|
|
4260
|
+
properties,
|
|
4261
|
+
builderModelFields,
|
|
4262
|
+
builderSampleEntries: builderEntries,
|
|
4263
|
+
existingFields,
|
|
4264
|
+
now: args.now,
|
|
4265
|
+
});
|
|
4266
|
+
}
|
|
3822
4267
|
|
|
3823
4268
|
const itemsToLink = response.items.filter((item) =>
|
|
3824
4269
|
builderEntriesByDocumentId.has(item.document.id),
|
|
3825
4270
|
);
|
|
3826
4271
|
const fetchedDocumentIds = itemsToLink.map((item) => item.document.id);
|
|
3827
4272
|
if (fetchedDocumentIds.length > 0) {
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
schema.contentDatabaseSourceRows.documentId,
|
|
3835
|
-
fetchedDocumentIds,
|
|
4273
|
+
for (const idChunk of chunks(fetchedDocumentIds, idChunkSize())) {
|
|
4274
|
+
await db
|
|
4275
|
+
.delete(schema.contentDatabaseSourceRows)
|
|
4276
|
+
.where(
|
|
4277
|
+
and(
|
|
4278
|
+
eq(schema.contentDatabaseSourceRows.sourceId, args.source.id),
|
|
4279
|
+
inArray(schema.contentDatabaseSourceRows.documentId, idChunk),
|
|
3836
4280
|
),
|
|
3837
|
-
)
|
|
3838
|
-
|
|
4281
|
+
);
|
|
4282
|
+
}
|
|
3839
4283
|
await seedMockSourceRows({
|
|
3840
4284
|
sourceId: args.source.id,
|
|
3841
4285
|
ownerEmail: args.database.ownerEmail,
|
|
@@ -3868,17 +4312,24 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
3868
4312
|
});
|
|
3869
4313
|
}
|
|
3870
4314
|
if (!hasMore && nextActiveReadSourceRowIds.length > 0) {
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
4315
|
+
const activeSourceRowIds = new Set(nextActiveReadSourceRowIds);
|
|
4316
|
+
const staleRows = (
|
|
4317
|
+
await db
|
|
4318
|
+
.select({
|
|
4319
|
+
id: schema.contentDatabaseSourceRows.id,
|
|
4320
|
+
sourceRowId: schema.contentDatabaseSourceRows.sourceRowId,
|
|
4321
|
+
})
|
|
4322
|
+
.from(schema.contentDatabaseSourceRows)
|
|
4323
|
+
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id))
|
|
4324
|
+
).filter((row) => !activeSourceRowIds.has(row.sourceRowId));
|
|
4325
|
+
for (const idChunk of chunks(
|
|
4326
|
+
staleRows.map((row) => row.id),
|
|
4327
|
+
idChunkSize(),
|
|
4328
|
+
)) {
|
|
4329
|
+
await db
|
|
4330
|
+
.delete(schema.contentDatabaseSourceRows)
|
|
4331
|
+
.where(inArray(schema.contentDatabaseSourceRows.id, idChunk));
|
|
4332
|
+
}
|
|
3882
4333
|
}
|
|
3883
4334
|
|
|
3884
4335
|
await updateBuilderCmsSourceReadMetadata({
|
|
@@ -3902,23 +4353,29 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
3902
4353
|
return;
|
|
3903
4354
|
}
|
|
3904
4355
|
|
|
3905
|
-
|
|
3906
|
-
.
|
|
3907
|
-
|
|
4356
|
+
const shouldReseedSourceFields =
|
|
4357
|
+
!builderModelFieldsReadFailed || existingFields.length === 0;
|
|
4358
|
+
if (shouldReseedSourceFields) {
|
|
4359
|
+
await db
|
|
4360
|
+
.delete(schema.contentDatabaseSourceFields)
|
|
4361
|
+
.where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
|
|
4362
|
+
}
|
|
3908
4363
|
await db
|
|
3909
4364
|
.delete(schema.contentDatabaseSourceRows)
|
|
3910
4365
|
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
|
|
3911
4366
|
|
|
3912
|
-
|
|
3913
|
-
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
|
|
4367
|
+
if (shouldReseedSourceFields) {
|
|
4368
|
+
await seedMockSourceFields({
|
|
4369
|
+
sourceId: args.source.id,
|
|
4370
|
+
ownerEmail: args.database.ownerEmail,
|
|
4371
|
+
sourceType: "builder-cms",
|
|
4372
|
+
properties,
|
|
4373
|
+
builderModelFields,
|
|
4374
|
+
builderSampleEntries: builderEntries,
|
|
4375
|
+
existingFields,
|
|
4376
|
+
now: args.now,
|
|
4377
|
+
});
|
|
4378
|
+
}
|
|
3922
4379
|
// Row-union: a resync must only (re)link items that BELONG to this source —
|
|
3923
4380
|
// never claim every database item. With a single source, all items belong to
|
|
3924
4381
|
// it (back-compat). With multiple sources, link only this source's
|