@agent-native/core 0.91.2 → 0.92.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (538) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +31 -0
  3. package/corpus/core/docs/content/faq.mdx +17 -0
  4. package/corpus/core/docs/content/notifications.mdx +12 -6
  5. package/corpus/core/package.json +1 -1
  6. package/corpus/core/src/agent/production-agent.ts +140 -46
  7. package/corpus/core/src/agent/tool-search.ts +71 -4
  8. package/corpus/core/src/application-state/store.ts +12 -1
  9. package/corpus/core/src/cli/index.ts +20 -0
  10. package/corpus/core/src/cli/upgrade.ts +798 -0
  11. package/corpus/core/src/client/AssistantChat.tsx +28 -15
  12. package/corpus/core/src/client/HighlightedCodeBlock.tsx +20 -3
  13. package/corpus/core/src/client/PoweredByBadge.tsx +1 -8
  14. package/corpus/core/src/client/agent-chat-adapter.ts +68 -16
  15. package/corpus/core/src/client/chat/message-components.tsx +85 -8
  16. package/corpus/core/src/client/chat/tool-call-display.tsx +355 -144
  17. package/corpus/core/src/client/chat/widgets/DataTableWidget.tsx +52 -1
  18. package/corpus/core/src/client/code-agent-chat-adapter.ts +2 -9
  19. package/corpus/core/src/client/db-admin/TableEditor.tsx +49 -0
  20. package/corpus/core/src/client/db-admin/export-utils.ts +13 -4
  21. package/corpus/core/src/client/db-admin/useAgentSync.ts +72 -0
  22. package/corpus/core/src/client/error-capture.ts +69 -7
  23. package/corpus/core/src/client/extensions/EmbeddedExtension.tsx +5 -1
  24. package/corpus/core/src/client/extensions/ExtensionViewer.tsx +53 -8
  25. package/corpus/core/src/client/extensions/InlineExtensionFrame.tsx +5 -1
  26. package/corpus/core/src/client/extensions/portable-extension.ts +1 -0
  27. package/corpus/core/src/client/org/OrgSwitcher.tsx +12 -9
  28. package/corpus/core/src/client/org/TeamPage.tsx +77 -71
  29. package/corpus/core/src/client/session-replay.ts +3 -97
  30. package/corpus/core/src/client/settings/SettingsPanel.tsx +738 -441
  31. package/corpus/core/src/client/settings/SettingsSection.tsx +161 -17
  32. package/corpus/core/src/client/settings/SettingsTabsPage.tsx +25 -9
  33. package/corpus/core/src/client/sse-event-processor.ts +29 -2
  34. package/corpus/core/src/db-admin/agent-tools.ts +8 -2
  35. package/corpus/core/src/db-admin/operations.ts +116 -4
  36. package/corpus/core/src/db-admin/routes.ts +1 -0
  37. package/corpus/core/src/db-admin/types.ts +4 -0
  38. package/corpus/core/src/extensions/actions.ts +113 -5
  39. package/corpus/core/src/extensions/html-shell.ts +5 -4
  40. package/corpus/core/src/extensions/routes.ts +10 -0
  41. package/corpus/core/src/navigation/index.ts +1 -1
  42. package/corpus/core/src/notifications/channels.ts +88 -25
  43. package/corpus/core/src/notifications/registry.ts +17 -3
  44. package/corpus/core/src/notifications/types.ts +4 -1
  45. package/corpus/core/src/resources/handlers.ts +10 -8
  46. package/corpus/core/src/server/agent-chat-plugin.ts +7 -3
  47. package/corpus/core/src/server/poll.ts +293 -8
  48. package/corpus/core/src/server/request-context.ts +7 -0
  49. package/corpus/core/src/styles/agent-conversation.css +1 -1
  50. package/corpus/core/src/styles/agent-native.css +27 -0
  51. package/corpus/core/src/templates/default/.agents/skills/actions/SKILL.md +5 -0
  52. package/corpus/core/src/templates/default/.agents/skills/adding-a-feature/SKILL.md +5 -0
  53. package/corpus/core/src/templates/default/.agents/skills/real-time-sync/SKILL.md +1 -1
  54. package/corpus/core/src/templates/default/.agents/skills/security/SKILL.md +7 -0
  55. package/corpus/core/src/templates/default/.agents/skills/self-modifying-code/SKILL.md +20 -2
  56. package/corpus/core/src/templates/default/.agents/skills/storing-data/SKILL.md +5 -1
  57. package/corpus/core/src/templates/default/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  58. package/corpus/core/src/templates/default/AGENTS.md +14 -5
  59. package/corpus/core/src/templates/default/package.json +2 -1
  60. package/corpus/core/src/templates/headless/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  61. package/corpus/core/src/templates/headless/AGENTS.md +13 -3
  62. package/corpus/core/src/templates/headless/package.json +1 -0
  63. package/corpus/core/src/templates/workspace-core/.agents/skills/actions/SKILL.md +5 -0
  64. package/corpus/core/src/templates/workspace-core/.agents/skills/adding-a-feature/SKILL.md +5 -0
  65. package/corpus/core/src/templates/workspace-core/.agents/skills/context-awareness/SKILL.md +4 -0
  66. package/corpus/core/src/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -0
  67. package/corpus/core/src/templates/workspace-core/.agents/skills/performance/SKILL.md +4 -0
  68. package/corpus/core/src/templates/workspace-core/.agents/skills/real-time-sync/SKILL.md +1 -1
  69. package/corpus/core/src/templates/workspace-core/.agents/skills/security/SKILL.md +7 -0
  70. package/corpus/core/src/templates/workspace-core/.agents/skills/self-modifying-code/SKILL.md +20 -2
  71. package/corpus/core/src/templates/workspace-core/.agents/skills/storing-data/SKILL.md +5 -1
  72. package/corpus/core/src/templates/workspace-core/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  73. package/corpus/core/src/templates/workspace-core/AGENTS.md +7 -1
  74. package/corpus/core/src/templates/workspace-root/AGENTS.md +17 -5
  75. package/corpus/core/src/templates/workspace-root/package.json +1 -0
  76. package/corpus/templates/analytics/.agents/skills/actions/SKILL.md +5 -0
  77. package/corpus/templates/analytics/.agents/skills/dashboard-management/SKILL.md +10 -0
  78. package/corpus/templates/analytics/.agents/skills/real-time-sync/SKILL.md +1 -1
  79. package/corpus/templates/analytics/.agents/skills/security/SKILL.md +7 -0
  80. package/corpus/templates/analytics/.agents/skills/self-modifying-code/SKILL.md +20 -2
  81. package/corpus/templates/analytics/.agents/skills/session-replay/SKILL.md +5 -6
  82. package/corpus/templates/analytics/.agents/skills/storing-data/SKILL.md +5 -1
  83. package/corpus/templates/analytics/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  84. package/corpus/templates/analytics/AGENTS.md +39 -18
  85. package/corpus/templates/analytics/actions/capture-test-error.ts +4 -0
  86. package/corpus/templates/analytics/actions/get-analytics-alert-rule-defaults.ts +23 -0
  87. package/corpus/templates/analytics/actions/get-error-issue.ts +1 -1
  88. package/corpus/templates/analytics/actions/list-analysis-revisions.ts +40 -0
  89. package/corpus/templates/analytics/actions/list-dashboard-revisions.ts +40 -0
  90. package/corpus/templates/analytics/actions/list-dashboard-usage-stats.ts +299 -0
  91. package/corpus/templates/analytics/actions/navigate.ts +4 -2
  92. package/corpus/templates/analytics/actions/restore-analysis-revision.ts +43 -0
  93. package/corpus/templates/analytics/actions/restore-dashboard-revision.ts +68 -0
  94. package/corpus/templates/analytics/actions/save-analytics-alert-rule.ts +14 -0
  95. package/corpus/templates/analytics/actions/save-monitor.ts +15 -1
  96. package/corpus/templates/analytics/actions/view-screen.ts +24 -0
  97. package/corpus/templates/analytics/app/components/analysis/AnalysisHistoryPanel.tsx +181 -0
  98. package/corpus/templates/analytics/app/components/dashboard/DashboardHistoryPanel.tsx +184 -0
  99. package/corpus/templates/analytics/app/components/dashboard/DataTable.tsx +41 -6
  100. package/corpus/templates/analytics/app/components/dashboard/SqlChart.tsx +122 -11
  101. package/corpus/templates/analytics/app/components/layout/CommandPalette.tsx +22 -16
  102. package/corpus/templates/analytics/app/components/layout/Header.tsx +1 -1
  103. package/corpus/templates/analytics/app/components/layout/Sidebar.tsx +27 -21
  104. package/corpus/templates/analytics/app/hooks/use-analysis-revisions.ts +43 -0
  105. package/corpus/templates/analytics/app/hooks/use-dashboard-chat-context.ts +110 -0
  106. package/corpus/templates/analytics/app/hooks/use-dashboard-revisions.ts +43 -0
  107. package/corpus/templates/analytics/app/hooks/use-navigation-state.ts +8 -3
  108. package/corpus/templates/analytics/app/i18n/zh-TW.ts +64 -2
  109. package/corpus/templates/analytics/app/i18n-data.ts +783 -3
  110. package/corpus/templates/analytics/app/pages/Agents.tsx +359 -19
  111. package/corpus/templates/analytics/app/pages/Settings.tsx +3 -2
  112. package/corpus/templates/analytics/app/pages/adhoc/explorer-dashboard/index.tsx +34 -1
  113. package/corpus/templates/analytics/app/pages/adhoc/sql-dashboard/index.tsx +28 -0
  114. package/corpus/templates/analytics/app/pages/analyses/AnalysisDetail.tsx +18 -1
  115. package/corpus/templates/analytics/app/pages/monitoring/ErrorsPanel.tsx +7 -10
  116. package/corpus/templates/analytics/app/pages/monitoring/MonitoringPage.tsx +8 -2
  117. package/corpus/templates/analytics/app/pages/monitoring/errors/IssueDetail.tsx +479 -23
  118. package/corpus/templates/analytics/app/pages/monitoring/errors/IssueList.tsx +0 -9
  119. package/corpus/templates/analytics/app/pages/monitoring/errors/i18n.ts +11 -0
  120. package/corpus/templates/analytics/app/pages/monitoring/errors/types.ts +9 -1
  121. package/corpus/templates/analytics/app/pages/monitoring/uptime/MonitorFormPage.tsx +157 -35
  122. package/corpus/templates/analytics/app/pages/monitoring/uptime/i18n.ts +17 -2
  123. package/corpus/templates/analytics/app/pages/monitoring/uptime/types.ts +4 -0
  124. package/corpus/templates/analytics/app/pages/sessions/SessionDetailPage.tsx +80 -220
  125. package/corpus/templates/analytics/app/pages/sessions/SessionsPage.tsx +9 -99
  126. package/corpus/templates/analytics/app/pages/settings/AlertRulesSettingsCard.tsx +262 -104
  127. package/corpus/templates/analytics/app/routes/team.tsx +1 -1
  128. package/corpus/templates/analytics/changelog/2026-07-08-admins-can-audit-dashboard-usage-ownership-and-cleanup-signa.md +6 -0
  129. package/corpus/templates/analytics/changelog/2026-07-08-agent-monitoring-stays-available-to-non-admins.md +6 -0
  130. package/corpus/templates/analytics/changelog/2026-07-08-alert-rules-are-easier-to-scan-expand-and-edit-from-settings.md +6 -0
  131. package/corpus/templates/analytics/changelog/2026-07-08-analytics-alert-forms-remember-last-email-recipients.md +6 -0
  132. package/corpus/templates/analytics/changelog/2026-07-08-analytics-alert-rules-can-now-use-per-rule-slack-and-webhook.md +6 -0
  133. package/corpus/templates/analytics/changelog/2026-07-08-dashboard-history-and-chat-context.md +5 -0
  134. package/corpus/templates/analytics/changelog/2026-07-08-dashboard-tables-keep-their-size-while-loading.md +6 -0
  135. package/corpus/templates/analytics/changelog/2026-07-08-dashboard-usage-now-counts-explorer-dashboard-traffic.md +6 -0
  136. package/corpus/templates/analytics/changelog/2026-07-08-demo-mode-session-emails-are-filtered-and-anonymized.md +6 -0
  137. package/corpus/templates/analytics/changelog/2026-07-08-error-issue-detail-adds-stack-traces-and-frequency-bars.md +6 -0
  138. package/corpus/templates/analytics/changelog/2026-07-08-error-issue-detail-now-shows-the-latest-occurrence-message-a.md +6 -0
  139. package/corpus/templates/analytics/changelog/2026-07-08-monitoring-errors-only-show-test-action-when-empty.md +6 -0
  140. package/corpus/templates/analytics/changelog/2026-07-08-monitoring-uptime-and-errors-tabs-now-clearly-show-which-view-is-active.md +6 -0
  141. package/corpus/templates/analytics/changelog/2026-07-08-production-session-replay-now-requires-private-blob-storage-.md +6 -0
  142. package/corpus/templates/analytics/changelog/2026-07-08-session-replay-uses-stock-rrweb-sizing-again-so-recorded.md +6 -0
  143. package/corpus/templates/analytics/changelog/2026-07-08-shared-extension-links-now-show-a-clear-message-when-you-do-.md +6 -0
  144. package/corpus/templates/analytics/changelog/2026-07-08-sidebar-dashboard-menus-now-open-above-the-dashboard-canvas.md +6 -0
  145. package/corpus/templates/analytics/changelog/2026-07-08-team-settings-show-access-controls-next-to-member-lists.md +6 -0
  146. package/corpus/templates/analytics/changelog/2026-07-08-uptime-alerting-clarifies-in-app-inbox-vs-email-and-lets-you-paste-slack-or-webhook-urls-per-monitor.md +6 -0
  147. package/corpus/templates/analytics/changelog/2026-07-08-uptime-monitors-now-use-a-10-second-request-timeout-by-defau.md +6 -0
  148. package/corpus/templates/analytics/docs/uptime-monitoring.md +13 -11
  149. package/corpus/templates/analytics/server/db/schema-monitoring.ts +5 -1
  150. package/corpus/templates/analytics/server/db/schema.ts +56 -0
  151. package/corpus/templates/analytics/server/lib/analytics-alerts.ts +92 -0
  152. package/corpus/templates/analytics/server/lib/dashboards-store.ts +334 -5
  153. package/corpus/templates/analytics/server/lib/error-capture.ts +231 -36
  154. package/corpus/templates/analytics/server/lib/session-replay.ts +114 -8
  155. package/corpus/templates/analytics/server/lib/uptime-monitors.ts +65 -16
  156. package/corpus/templates/analytics/server/plugins/db.ts +122 -2
  157. package/corpus/templates/analytics/server/routes/api/dashboard-agent-context.json.get.ts +1 -0
  158. package/corpus/templates/assets/.agents/skills/actions/SKILL.md +5 -0
  159. package/corpus/templates/assets/.agents/skills/real-time-sync/SKILL.md +1 -1
  160. package/corpus/templates/assets/.agents/skills/security/SKILL.md +7 -0
  161. package/corpus/templates/assets/.agents/skills/self-modifying-code/SKILL.md +20 -2
  162. package/corpus/templates/assets/.agents/skills/storing-data/SKILL.md +5 -1
  163. package/corpus/templates/assets/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  164. package/corpus/templates/assets/AGENTS.md +4 -0
  165. package/corpus/templates/assets/app/routes/settings.tsx +1 -1
  166. package/corpus/templates/assets/app/routes/team.tsx +1 -1
  167. package/corpus/templates/brain/.agents/skills/actions/SKILL.md +5 -0
  168. package/corpus/templates/brain/.agents/skills/adding-a-feature/SKILL.md +5 -0
  169. package/corpus/templates/brain/.agents/skills/real-time-sync/SKILL.md +1 -1
  170. package/corpus/templates/brain/.agents/skills/security/SKILL.md +7 -0
  171. package/corpus/templates/brain/.agents/skills/self-modifying-code/SKILL.md +20 -2
  172. package/corpus/templates/brain/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  173. package/corpus/templates/brain/AGENTS.md +4 -0
  174. package/corpus/templates/brain/app/routes/team.tsx +1 -1
  175. package/corpus/templates/calendar/.agents/skills/actions/SKILL.md +5 -0
  176. package/corpus/templates/calendar/.agents/skills/real-time-sync/SKILL.md +1 -1
  177. package/corpus/templates/calendar/.agents/skills/security/SKILL.md +7 -0
  178. package/corpus/templates/calendar/.agents/skills/self-modifying-code/SKILL.md +20 -2
  179. package/corpus/templates/calendar/.agents/skills/storing-data/SKILL.md +5 -1
  180. package/corpus/templates/calendar/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  181. package/corpus/templates/calendar/AGENTS.md +4 -0
  182. package/corpus/templates/calendar/app/components/calendar/DayView.tsx +11 -2
  183. package/corpus/templates/calendar/app/components/calendar/EventDetailPopover.tsx +21 -8
  184. package/corpus/templates/calendar/app/components/calendar/MonthView.tsx +7 -1
  185. package/corpus/templates/calendar/app/components/calendar/WeekView.tsx +11 -2
  186. package/corpus/templates/calendar/app/lib/popover-click-guard.ts +48 -4
  187. package/corpus/templates/calendar/app/pages/Settings.tsx +1 -1
  188. package/corpus/templates/calendar/app/routes/_app.team.tsx +1 -1
  189. package/corpus/templates/calendar/changelog/2026-07-08-clicking-empty-calendar-space-closes-an-open-event-popover-b.md +6 -0
  190. package/corpus/templates/chat/.agents/skills/actions/SKILL.md +5 -0
  191. package/corpus/templates/chat/.agents/skills/adding-a-feature/SKILL.md +5 -0
  192. package/corpus/templates/chat/.agents/skills/real-time-sync/SKILL.md +1 -1
  193. package/corpus/templates/chat/.agents/skills/security/SKILL.md +7 -0
  194. package/corpus/templates/chat/.agents/skills/self-modifying-code/SKILL.md +20 -2
  195. package/corpus/templates/chat/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  196. package/corpus/templates/chat/AGENTS.md +4 -0
  197. package/corpus/templates/chat/app/hooks/use-navigation-state.ts +1 -1
  198. package/corpus/templates/chat/app/routes/settings.tsx +2 -2
  199. package/corpus/templates/chat/app/routes/team.tsx +1 -1
  200. package/corpus/templates/clips/.agents/skills/actions/SKILL.md +5 -0
  201. package/corpus/templates/clips/.agents/skills/security/SKILL.md +7 -0
  202. package/corpus/templates/clips/.agents/skills/self-modifying-code/SKILL.md +20 -2
  203. package/corpus/templates/clips/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  204. package/corpus/templates/clips/AGENTS.md +4 -0
  205. package/corpus/templates/clips/actions/finalize-recording.ts +9 -3
  206. package/corpus/templates/clips/actions/set-thumbnail.ts +11 -2
  207. package/corpus/templates/clips/app/components/meetings/canvas-editor.tsx +1 -7
  208. package/corpus/templates/clips/app/components/player/player-controls.tsx +21 -13
  209. package/corpus/templates/clips/app/components/player/video-player.tsx +30 -75
  210. package/corpus/templates/clips/app/i18n/ar-SA.ts +6 -5
  211. package/corpus/templates/clips/app/i18n/de-DE.ts +6 -6
  212. package/corpus/templates/clips/app/i18n/en-US.ts +6 -5
  213. package/corpus/templates/clips/app/i18n/es-ES.ts +6 -5
  214. package/corpus/templates/clips/app/i18n/fr-FR.ts +6 -6
  215. package/corpus/templates/clips/app/i18n/hi-IN.ts +6 -5
  216. package/corpus/templates/clips/app/i18n/ja-JP.ts +6 -5
  217. package/corpus/templates/clips/app/i18n/ko-KR.ts +6 -5
  218. package/corpus/templates/clips/app/i18n/pt-BR.ts +6 -5
  219. package/corpus/templates/clips/app/i18n/zh-CN.ts +6 -5
  220. package/corpus/templates/clips/app/i18n/zh-TW.ts +6 -5
  221. package/corpus/templates/clips/app/routes/_app.meetings.$meetingId.tsx +51 -118
  222. package/corpus/templates/clips/changelog/2026-07-08-desktop-meeting-notes-stay-active-from-tray-and-shortcut-controls.md +6 -0
  223. package/corpus/templates/clips/changelog/2026-07-08-desktop-meeting-popovers-can-start-notes-and-join-the-meetin.md +6 -0
  224. package/corpus/templates/clips/changelog/2026-07-08-hosted-clip-uploads-now-clear-failed-buffered-chunks.md +6 -0
  225. package/corpus/templates/clips/changelog/2026-07-08-hosted-recording-uploads-now-avoid-sql-chunk-storage.md +6 -0
  226. package/corpus/templates/clips/changelog/2026-07-08-meeting-notes-keep-their-recording-state-when-the-desktop.md +6 -0
  227. package/corpus/templates/clips/changelog/2026-07-08-meeting-recaps-now-keep-summaries-action-items-and-transcrip.md +6 -0
  228. package/corpus/templates/clips/changelog/2026-07-08-meeting-reminder-dropdowns-now-expand-fully-instead-of-being.md +6 -0
  229. package/corpus/templates/clips/changelog/2026-07-08-video-playback-controls-now-match-loom-style-5-second-skips-.md +6 -0
  230. package/corpus/templates/clips/desktop/src/app.tsx +123 -3
  231. package/corpus/templates/clips/desktop/src/hooks/useMeetingTranscription.ts +15 -1
  232. package/corpus/templates/clips/desktop/src/overlays/meeting-notification.tsx +23 -0
  233. package/corpus/templates/clips/desktop/src/styles.css +23 -1
  234. package/corpus/templates/clips/desktop/src-tauri/src/adhoc_meetings_watcher.rs +1 -3
  235. package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +40 -6
  236. package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +4 -2
  237. package/corpus/templates/clips/desktop/src-tauri/src/meetings_watcher.rs +3 -5
  238. package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +23 -19
  239. package/corpus/templates/clips/desktop/src-tauri/src/shortcuts.rs +3 -3
  240. package/corpus/templates/clips/desktop/src-tauri/src/silence_detector.rs +0 -1
  241. package/corpus/templates/clips/desktop/src-tauri/src/state.rs +5 -0
  242. package/corpus/templates/clips/desktop/src-tauri/src/tray.rs +18 -6
  243. package/corpus/templates/clips/server/lib/recording-upload-state.ts +63 -0
  244. package/corpus/templates/clips/server/lib/video-storage.ts +4 -0
  245. package/corpus/templates/clips/server/routes/api/recordings/[recordingId]/thumbnail.post.ts +19 -13
  246. package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/chunk.post.ts +40 -1
  247. package/corpus/templates/content/.agents/skills/actions/SKILL.md +5 -0
  248. package/corpus/templates/content/.agents/skills/real-time-sync/SKILL.md +1 -1
  249. package/corpus/templates/content/.agents/skills/security/SKILL.md +7 -0
  250. package/corpus/templates/content/.agents/skills/self-modifying-code/SKILL.md +20 -2
  251. package/corpus/templates/content/.agents/skills/storing-data/SKILL.md +5 -1
  252. package/corpus/templates/content/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  253. package/corpus/templates/content/AGENTS.md +4 -0
  254. package/corpus/templates/content/actions/_batch-utils.ts +8 -0
  255. package/corpus/templates/content/actions/_builder-cms-read-client.ts +7 -5
  256. package/corpus/templates/content/actions/_database-source-utils.ts +776 -319
  257. package/corpus/templates/content/actions/_database-utils.ts +6 -1
  258. package/corpus/templates/content/actions/_property-utils.ts +168 -0
  259. package/corpus/templates/content/actions/add-content-database-source-field-property.ts +19 -12
  260. package/corpus/templates/content/actions/bind-content-database-source-field.ts +19 -12
  261. package/corpus/templates/content/actions/delete-document.ts +9 -13
  262. package/corpus/templates/content/actions/refresh-content-database-source.ts +5 -5
  263. package/corpus/templates/content/app/components/editor/body-hydration.ts +35 -11
  264. package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +47 -5
  265. package/corpus/templates/content/app/hooks/use-documents.ts +8 -2
  266. package/corpus/templates/content/app/routes/_app.settings.tsx +2 -2
  267. package/corpus/templates/content/app/routes/_app.team.tsx +1 -1
  268. package/corpus/templates/content/changelog/2026-07-08-builder-source-sync-now-completes-reliably-on-hosted-databases.md +6 -0
  269. package/corpus/templates/content/changelog/2026-07-08-fixed-builder-database-sync-so-large-sources-continue-past-t.md +6 -0
  270. package/corpus/templates/content/server/plugins/db.ts +19 -0
  271. package/corpus/templates/design/.agents/skills/actions/SKILL.md +5 -0
  272. package/corpus/templates/design/.agents/skills/security/SKILL.md +7 -0
  273. package/corpus/templates/design/.agents/skills/self-modifying-code/SKILL.md +20 -2
  274. package/corpus/templates/design/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  275. package/corpus/templates/design/AGENTS.md +4 -0
  276. package/corpus/templates/design/actions/take-design-screenshot.ts +12 -9
  277. package/corpus/templates/design/app/components/design/inspector/ImageFillControls.tsx +57 -13
  278. package/corpus/templates/design/app/routes/team.tsx +1 -1
  279. package/corpus/templates/design/changelog/2026-07-08-local-images-added-to-designs-now-upload-to-file-storage-ins.md +6 -0
  280. package/corpus/templates/dispatch/.agents/skills/actions/SKILL.md +5 -0
  281. package/corpus/templates/dispatch/.agents/skills/security/SKILL.md +7 -0
  282. package/corpus/templates/dispatch/.agents/skills/self-modifying-code/SKILL.md +20 -2
  283. package/corpus/templates/dispatch/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  284. package/corpus/templates/dispatch/AGENTS.md +4 -0
  285. package/corpus/templates/dispatch/app/hooks/use-navigation-state.ts +1 -1
  286. package/corpus/templates/dispatch/app/routes/settings.tsx +2 -2
  287. package/corpus/templates/dispatch/app/routes/team.tsx +1 -1
  288. package/corpus/templates/forms/.agents/skills/actions/SKILL.md +5 -0
  289. package/corpus/templates/forms/.agents/skills/real-time-sync/SKILL.md +1 -1
  290. package/corpus/templates/forms/.agents/skills/security/SKILL.md +7 -0
  291. package/corpus/templates/forms/.agents/skills/self-modifying-code/SKILL.md +20 -2
  292. package/corpus/templates/forms/.agents/skills/storing-data/SKILL.md +5 -1
  293. package/corpus/templates/forms/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  294. package/corpus/templates/forms/AGENTS.md +4 -0
  295. package/corpus/templates/forms/app/routes/_app.settings.tsx +2 -2
  296. package/corpus/templates/forms/app/routes/_app.team.tsx +1 -1
  297. package/corpus/templates/forms/shared/navigation.ts +1 -1
  298. package/corpus/templates/macros/.agents/skills/security/SKILL.md +7 -0
  299. package/corpus/templates/macros/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  300. package/corpus/templates/macros/AGENTS.md +4 -0
  301. package/corpus/templates/macros/app/routes/settings.tsx +2 -2
  302. package/corpus/templates/macros/app/routes/team.tsx +1 -1
  303. package/corpus/templates/mail/.agents/skills/actions/SKILL.md +5 -0
  304. package/corpus/templates/mail/.agents/skills/email-drafts/SKILL.md +1 -1
  305. package/corpus/templates/mail/.agents/skills/real-time-sync/SKILL.md +1 -1
  306. package/corpus/templates/mail/.agents/skills/security/SKILL.md +7 -0
  307. package/corpus/templates/mail/.agents/skills/self-modifying-code/SKILL.md +20 -2
  308. package/corpus/templates/mail/.agents/skills/storing-data/SKILL.md +5 -1
  309. package/corpus/templates/mail/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  310. package/corpus/templates/mail/AGENTS.md +4 -0
  311. package/corpus/templates/mail/changelog/2026-07-08-mail-attachments-in-hosted-environments-now-require-file-sto.md +6 -0
  312. package/corpus/templates/mail/server/handlers/media.ts +26 -1
  313. package/corpus/templates/mail/server/lib/outgoing-email.ts +24 -1
  314. package/corpus/templates/mail/server/lib/upload-store.ts +2 -1
  315. package/corpus/templates/plan/.agents/skills/actions/SKILL.md +5 -0
  316. package/corpus/templates/plan/.agents/skills/adding-a-feature/SKILL.md +5 -0
  317. package/corpus/templates/plan/.agents/skills/real-time-sync/SKILL.md +1 -1
  318. package/corpus/templates/plan/.agents/skills/security/SKILL.md +7 -0
  319. package/corpus/templates/plan/.agents/skills/self-modifying-code/SKILL.md +20 -2
  320. package/corpus/templates/plan/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  321. package/corpus/templates/plan/AGENTS.md +4 -0
  322. package/corpus/templates/plan/app/hooks/use-navigation-state.ts +1 -1
  323. package/corpus/templates/plan/app/routes/settings.tsx +2 -2
  324. package/corpus/templates/plan/app/routes/team.tsx +1 -1
  325. package/corpus/templates/plan/changelog/2026-07-08-hosted-plan-images-now-require-connected-storage.md +6 -0
  326. package/corpus/templates/plan/server/lib/plan-assets.ts +33 -4
  327. package/corpus/templates/slides/.agents/skills/actions/SKILL.md +5 -0
  328. package/corpus/templates/slides/.agents/skills/security/SKILL.md +7 -0
  329. package/corpus/templates/slides/.agents/skills/self-modifying-code/SKILL.md +20 -2
  330. package/corpus/templates/slides/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  331. package/corpus/templates/slides/AGENTS.md +4 -0
  332. package/corpus/templates/slides/actions/generate-image-api.ts +15 -3
  333. package/corpus/templates/slides/app/routes/settings.tsx +2 -2
  334. package/corpus/templates/slides/app/routes/team.tsx +1 -1
  335. package/corpus/templates/slides/changelog/2026-07-08-generated-slide-images-now-return-hosted-file-storage-urls-i.md +6 -0
  336. package/corpus/templates/slides/shared/api.ts +1 -1
  337. package/corpus/templates/videos/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  338. package/corpus/templates/voice/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  339. package/dist/agent/production-agent.d.ts +5 -1
  340. package/dist/agent/production-agent.d.ts.map +1 -1
  341. package/dist/agent/production-agent.js +112 -42
  342. package/dist/agent/production-agent.js.map +1 -1
  343. package/dist/agent/tool-search.d.ts +2 -0
  344. package/dist/agent/tool-search.d.ts.map +1 -1
  345. package/dist/agent/tool-search.js +60 -4
  346. package/dist/agent/tool-search.js.map +1 -1
  347. package/dist/application-state/store.d.ts.map +1 -1
  348. package/dist/application-state/store.js +8 -2
  349. package/dist/application-state/store.js.map +1 -1
  350. package/dist/cli/index.js +19 -0
  351. package/dist/cli/index.js.map +1 -1
  352. package/dist/cli/upgrade.d.ts +100 -0
  353. package/dist/cli/upgrade.d.ts.map +1 -0
  354. package/dist/cli/upgrade.js +678 -0
  355. package/dist/cli/upgrade.js.map +1 -0
  356. package/dist/client/AssistantChat.d.ts.map +1 -1
  357. package/dist/client/AssistantChat.js +25 -14
  358. package/dist/client/AssistantChat.js.map +1 -1
  359. package/dist/client/HighlightedCodeBlock.d.ts +2 -0
  360. package/dist/client/HighlightedCodeBlock.d.ts.map +1 -1
  361. package/dist/client/HighlightedCodeBlock.js +10 -1
  362. package/dist/client/HighlightedCodeBlock.js.map +1 -1
  363. package/dist/client/PoweredByBadge.d.ts +1 -1
  364. package/dist/client/PoweredByBadge.d.ts.map +1 -1
  365. package/dist/client/PoweredByBadge.js +3 -9
  366. package/dist/client/PoweredByBadge.js.map +1 -1
  367. package/dist/client/agent-chat-adapter.d.ts.map +1 -1
  368. package/dist/client/agent-chat-adapter.js +60 -13
  369. package/dist/client/agent-chat-adapter.js.map +1 -1
  370. package/dist/client/chat/message-components.d.ts.map +1 -1
  371. package/dist/client/chat/message-components.js +61 -8
  372. package/dist/client/chat/message-components.js.map +1 -1
  373. package/dist/client/chat/tool-call-display.d.ts +12 -0
  374. package/dist/client/chat/tool-call-display.d.ts.map +1 -1
  375. package/dist/client/chat/tool-call-display.js +136 -42
  376. package/dist/client/chat/tool-call-display.js.map +1 -1
  377. package/dist/client/chat/widgets/DataTableWidget.d.ts.map +1 -1
  378. package/dist/client/chat/widgets/DataTableWidget.js +19 -2
  379. package/dist/client/chat/widgets/DataTableWidget.js.map +1 -1
  380. package/dist/client/code-agent-chat-adapter.js +2 -9
  381. package/dist/client/code-agent-chat-adapter.js.map +1 -1
  382. package/dist/client/db-admin/TableEditor.d.ts.map +1 -1
  383. package/dist/client/db-admin/TableEditor.js +19 -2
  384. package/dist/client/db-admin/TableEditor.js.map +1 -1
  385. package/dist/client/db-admin/export-utils.d.ts +2 -0
  386. package/dist/client/db-admin/export-utils.d.ts.map +1 -1
  387. package/dist/client/db-admin/export-utils.js +11 -3
  388. package/dist/client/db-admin/export-utils.js.map +1 -1
  389. package/dist/client/db-admin/useAgentSync.d.ts.map +1 -1
  390. package/dist/client/db-admin/useAgentSync.js +63 -0
  391. package/dist/client/db-admin/useAgentSync.js.map +1 -1
  392. package/dist/client/error-capture.d.ts.map +1 -1
  393. package/dist/client/error-capture.js +41 -7
  394. package/dist/client/error-capture.js.map +1 -1
  395. package/dist/client/extensions/EmbeddedExtension.d.ts.map +1 -1
  396. package/dist/client/extensions/EmbeddedExtension.js +3 -1
  397. package/dist/client/extensions/EmbeddedExtension.js.map +1 -1
  398. package/dist/client/extensions/ExtensionViewer.d.ts.map +1 -1
  399. package/dist/client/extensions/ExtensionViewer.js +32 -5
  400. package/dist/client/extensions/ExtensionViewer.js.map +1 -1
  401. package/dist/client/extensions/InlineExtensionFrame.d.ts.map +1 -1
  402. package/dist/client/extensions/InlineExtensionFrame.js +3 -1
  403. package/dist/client/extensions/InlineExtensionFrame.js.map +1 -1
  404. package/dist/client/extensions/portable-extension.d.ts.map +1 -1
  405. package/dist/client/extensions/portable-extension.js +1 -0
  406. package/dist/client/extensions/portable-extension.js.map +1 -1
  407. package/dist/client/org/OrgSwitcher.d.ts +1 -1
  408. package/dist/client/org/OrgSwitcher.d.ts.map +1 -1
  409. package/dist/client/org/OrgSwitcher.js +11 -6
  410. package/dist/client/org/OrgSwitcher.js.map +1 -1
  411. package/dist/client/org/TeamPage.d.ts.map +1 -1
  412. package/dist/client/org/TeamPage.js +5 -5
  413. package/dist/client/org/TeamPage.js.map +1 -1
  414. package/dist/client/session-replay.d.ts +0 -8
  415. package/dist/client/session-replay.d.ts.map +1 -1
  416. package/dist/client/session-replay.js +3 -80
  417. package/dist/client/session-replay.js.map +1 -1
  418. package/dist/client/settings/SettingsPanel.d.ts.map +1 -1
  419. package/dist/client/settings/SettingsPanel.js +89 -33
  420. package/dist/client/settings/SettingsPanel.js.map +1 -1
  421. package/dist/client/settings/SettingsSection.d.ts +19 -4
  422. package/dist/client/settings/SettingsSection.d.ts.map +1 -1
  423. package/dist/client/settings/SettingsSection.js +33 -4
  424. package/dist/client/settings/SettingsSection.js.map +1 -1
  425. package/dist/client/settings/SettingsTabsPage.d.ts.map +1 -1
  426. package/dist/client/settings/SettingsTabsPage.js +21 -10
  427. package/dist/client/settings/SettingsTabsPage.js.map +1 -1
  428. package/dist/client/sse-event-processor.d.ts +9 -0
  429. package/dist/client/sse-event-processor.d.ts.map +1 -1
  430. package/dist/client/sse-event-processor.js +20 -2
  431. package/dist/client/sse-event-processor.js.map +1 -1
  432. package/dist/db-admin/agent-tools.d.ts.map +1 -1
  433. package/dist/db-admin/agent-tools.js +7 -2
  434. package/dist/db-admin/agent-tools.js.map +1 -1
  435. package/dist/db-admin/operations.d.ts.map +1 -1
  436. package/dist/db-admin/operations.js +90 -4
  437. package/dist/db-admin/operations.js.map +1 -1
  438. package/dist/db-admin/routes.d.ts.map +1 -1
  439. package/dist/db-admin/routes.js +1 -0
  440. package/dist/db-admin/routes.js.map +1 -1
  441. package/dist/db-admin/types.d.ts +4 -0
  442. package/dist/db-admin/types.d.ts.map +1 -1
  443. package/dist/db-admin/types.js.map +1 -1
  444. package/dist/extensions/actions.d.ts.map +1 -1
  445. package/dist/extensions/actions.js +85 -5
  446. package/dist/extensions/actions.js.map +1 -1
  447. package/dist/extensions/html-shell.d.ts +5 -4
  448. package/dist/extensions/html-shell.d.ts.map +1 -1
  449. package/dist/extensions/html-shell.js.map +1 -1
  450. package/dist/extensions/routes.d.ts.map +1 -1
  451. package/dist/extensions/routes.js +8 -0
  452. package/dist/extensions/routes.js.map +1 -1
  453. package/dist/navigation/index.d.ts +1 -1
  454. package/dist/navigation/index.d.ts.map +1 -1
  455. package/dist/navigation/index.js +1 -1
  456. package/dist/navigation/index.js.map +1 -1
  457. package/dist/notifications/channels.d.ts +8 -3
  458. package/dist/notifications/channels.d.ts.map +1 -1
  459. package/dist/notifications/channels.js +61 -22
  460. package/dist/notifications/channels.js.map +1 -1
  461. package/dist/notifications/registry.d.ts.map +1 -1
  462. package/dist/notifications/registry.js +14 -3
  463. package/dist/notifications/registry.js.map +1 -1
  464. package/dist/notifications/types.d.ts +4 -1
  465. package/dist/notifications/types.d.ts.map +1 -1
  466. package/dist/notifications/types.js.map +1 -1
  467. package/dist/resources/handlers.d.ts +5 -0
  468. package/dist/resources/handlers.d.ts.map +1 -1
  469. package/dist/resources/handlers.js +9 -8
  470. package/dist/resources/handlers.js.map +1 -1
  471. package/dist/server/agent-chat-plugin.d.ts +5 -1
  472. package/dist/server/agent-chat-plugin.d.ts.map +1 -1
  473. package/dist/server/agent-chat-plugin.js +2 -2
  474. package/dist/server/agent-chat-plugin.js.map +1 -1
  475. package/dist/server/agent-engine-api-key-route.d.ts +1 -1
  476. package/dist/server/poll.d.ts +12 -4
  477. package/dist/server/poll.d.ts.map +1 -1
  478. package/dist/server/poll.js +236 -9
  479. package/dist/server/poll.js.map +1 -1
  480. package/dist/server/request-context.d.ts +7 -0
  481. package/dist/server/request-context.d.ts.map +1 -1
  482. package/dist/server/request-context.js.map +1 -1
  483. package/dist/styles/agent-conversation.css +1 -1
  484. package/dist/styles/agent-native.css +27 -0
  485. package/dist/templates/default/.agents/skills/actions/SKILL.md +5 -0
  486. package/dist/templates/default/.agents/skills/adding-a-feature/SKILL.md +5 -0
  487. package/dist/templates/default/.agents/skills/real-time-sync/SKILL.md +1 -1
  488. package/dist/templates/default/.agents/skills/security/SKILL.md +7 -0
  489. package/dist/templates/default/.agents/skills/self-modifying-code/SKILL.md +20 -2
  490. package/dist/templates/default/.agents/skills/storing-data/SKILL.md +5 -1
  491. package/dist/templates/default/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  492. package/dist/templates/default/AGENTS.md +14 -5
  493. package/dist/templates/default/package.json +2 -1
  494. package/dist/templates/headless/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  495. package/dist/templates/headless/AGENTS.md +13 -3
  496. package/dist/templates/headless/package.json +1 -0
  497. package/dist/templates/workspace-core/.agents/skills/actions/SKILL.md +5 -0
  498. package/dist/templates/workspace-core/.agents/skills/adding-a-feature/SKILL.md +5 -0
  499. package/dist/templates/workspace-core/.agents/skills/context-awareness/SKILL.md +4 -0
  500. package/dist/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -0
  501. package/dist/templates/workspace-core/.agents/skills/performance/SKILL.md +4 -0
  502. package/dist/templates/workspace-core/.agents/skills/real-time-sync/SKILL.md +1 -1
  503. package/dist/templates/workspace-core/.agents/skills/security/SKILL.md +7 -0
  504. package/dist/templates/workspace-core/.agents/skills/self-modifying-code/SKILL.md +20 -2
  505. package/dist/templates/workspace-core/.agents/skills/storing-data/SKILL.md +5 -1
  506. package/dist/templates/workspace-core/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  507. package/dist/templates/workspace-core/AGENTS.md +7 -1
  508. package/dist/templates/workspace-root/AGENTS.md +17 -5
  509. package/dist/templates/workspace-root/package.json +1 -0
  510. package/docs/content/faq.mdx +17 -0
  511. package/docs/content/notifications.mdx +12 -6
  512. package/package.json +2 -2
  513. package/src/templates/default/.agents/skills/actions/SKILL.md +5 -0
  514. package/src/templates/default/.agents/skills/adding-a-feature/SKILL.md +5 -0
  515. package/src/templates/default/.agents/skills/real-time-sync/SKILL.md +1 -1
  516. package/src/templates/default/.agents/skills/security/SKILL.md +7 -0
  517. package/src/templates/default/.agents/skills/self-modifying-code/SKILL.md +20 -2
  518. package/src/templates/default/.agents/skills/storing-data/SKILL.md +5 -1
  519. package/src/templates/default/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  520. package/src/templates/default/AGENTS.md +14 -5
  521. package/src/templates/default/package.json +2 -1
  522. package/src/templates/headless/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  523. package/src/templates/headless/AGENTS.md +13 -3
  524. package/src/templates/headless/package.json +1 -0
  525. package/src/templates/workspace-core/.agents/skills/actions/SKILL.md +5 -0
  526. package/src/templates/workspace-core/.agents/skills/adding-a-feature/SKILL.md +5 -0
  527. package/src/templates/workspace-core/.agents/skills/context-awareness/SKILL.md +4 -0
  528. package/src/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -0
  529. package/src/templates/workspace-core/.agents/skills/performance/SKILL.md +4 -0
  530. package/src/templates/workspace-core/.agents/skills/real-time-sync/SKILL.md +1 -1
  531. package/src/templates/workspace-core/.agents/skills/security/SKILL.md +7 -0
  532. package/src/templates/workspace-core/.agents/skills/self-modifying-code/SKILL.md +20 -2
  533. package/src/templates/workspace-core/.agents/skills/storing-data/SKILL.md +5 -1
  534. package/src/templates/workspace-core/.agents/skills/upgrade-agent-native/SKILL.md +87 -0
  535. package/src/templates/workspace-core/AGENTS.md +7 -1
  536. package/src/templates/workspace-root/AGENTS.md +17 -5
  537. package/src/templates/workspace-root/package.json +1 -0
  538. package/corpus/templates/analytics/changelog/2026-07-08-session-replay-clamps-ultra-wide-viewports-while-keeping-the.md +0 -6
@@ -1,5 +1,6 @@
1
1
  import { deleteCollabState, releaseDoc } from "@agent-native/core/collab";
2
- import { and, asc, eq, inArray, isNull, notInArray, sql } from "drizzle-orm";
2
+ import { getDialect, type Dialect } from "@agent-native/core/db";
3
+ import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm";
3
4
 
4
5
  import { getDb, schema } from "../server/db/index.js";
5
6
  import type {
@@ -50,6 +51,7 @@ import {
50
51
  type DocumentPropertyOptionColor,
51
52
  } from "../shared/properties.js";
52
53
  import { sanitizeNormalizationFormula } from "../shared/properties.js";
54
+ import { chunks } from "./_batch-utils.js";
53
55
  import {
54
56
  readBuilderCmsContentEntry,
55
57
  readBuilderCmsContentEntries,
@@ -531,12 +533,37 @@ function normalizeBuilderBodyBaselineContent(value: string | null | undefined) {
531
533
 
532
534
  const BUILDER_BODY_HYDRATION_BACKGROUND_PRIORITY = 10;
533
535
  const BUILDER_BODY_HYDRATION_OPEN_PRIORITY = 0;
534
- const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 8;
536
+ const BUILDER_BODY_HYDRATION_BATCH_LIMIT = 25;
537
+ const BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY = 6;
535
538
  const BUILDER_BODY_HYDRATION_MAX_ATTEMPTS = 5;
536
539
  const BUILDER_BODY_HYDRATION_CODEC_VERSION = "readable-native-images-v5";
537
540
  const BUILDER_CMS_REFRESH_INITIAL_PAGES = 1;
538
541
  const BUILDER_BODY_NOT_AVAILABLE_ERROR = "body not yet available from Builder";
539
542
 
543
+ export function bulkChunkSizeForColumnCount(
544
+ columnCount: number,
545
+ dialect: Dialect = getDialect(),
546
+ ) {
547
+ // D1 rejects statements with more than 100 bound params, so derive every
548
+ // bulk chunk from the statement's column count instead of a fixed row count.
549
+ const budget = dialect === "d1" ? 90 : 900;
550
+ return Math.max(1, Math.floor(budget / Math.max(1, columnCount)));
551
+ }
552
+
553
+ function idChunkSize() {
554
+ return bulkChunkSizeForColumnCount(1);
555
+ }
556
+
557
+ async function processInBatches<T>(
558
+ items: T[],
559
+ concurrency: number,
560
+ worker: (item: T) => Promise<void>,
561
+ ) {
562
+ for (const batch of chunks(items, concurrency)) {
563
+ await Promise.all(batch.map((item) => worker(item)));
564
+ }
565
+ }
566
+
540
567
  function builderBodyHydrationDelayMs() {
541
568
  if (
542
569
  process.env.NODE_ENV === "production" ||
@@ -801,7 +828,7 @@ async function readBuilderEntryWithLiveBodyFromSourceRow(args: {
801
828
  const liveEntry = await readBuilderCmsContentEntry({
802
829
  model: args.sourceTable,
803
830
  entryId: args.row.sourceRowId,
804
- }).catch(() => null);
831
+ });
805
832
  if (!liveEntry || liveEntry.id !== args.row.sourceRowId) return null;
806
833
  const entryWithStoredValues = {
807
834
  ...liveEntry,
@@ -828,62 +855,134 @@ export async function enqueueBuilderBodyHydration(args: {
828
855
  now: string;
829
856
  priority?: number;
830
857
  }) {
858
+ await enqueueBuilderBodyHydrations([args]);
859
+ }
860
+
861
+ type BuilderBodyHydrationEnqueueRequest = {
862
+ sourceId: string;
863
+ ownerEmail: string;
864
+ orgId: string | null;
865
+ databaseItemId: string;
866
+ documentId: string;
867
+ sourceTable: string;
868
+ entry: BuilderCmsSourceEntry;
869
+ now: string;
870
+ priority?: number;
871
+ };
872
+
873
+ async function enqueueBuilderBodyHydrations(
874
+ requests: BuilderBodyHydrationEnqueueRequest[],
875
+ ): Promise<ContentDatabaseBodyHydrationQueueRowDb[]> {
876
+ if (requests.length === 0) return [];
877
+ const uniqueRequestsByItemId = new Map<
878
+ string,
879
+ BuilderBodyHydrationEnqueueRequest
880
+ >();
881
+ for (const request of requests) {
882
+ const existing = uniqueRequestsByItemId.get(request.databaseItemId);
883
+ if (!existing) {
884
+ uniqueRequestsByItemId.set(request.databaseItemId, request);
885
+ continue;
886
+ }
887
+ const existingPriority =
888
+ existing.priority ??
889
+ builderBodyHydrationPriorityForRequest({ documentId: null });
890
+ const nextPriority =
891
+ request.priority ??
892
+ builderBodyHydrationPriorityForRequest({ documentId: null });
893
+ uniqueRequestsByItemId.set(request.databaseItemId, {
894
+ ...request,
895
+ priority: Math.min(existingPriority, nextPriority),
896
+ });
897
+ }
898
+ const uniqueRequests = Array.from(uniqueRequestsByItemId.values());
831
899
  const db = getDb();
832
- const sourceRowId = args.entry.id;
833
- const priority =
834
- args.priority ??
835
- builderBodyHydrationPriorityForRequest({ documentId: null });
836
- const [existing] = await db
837
- .select()
838
- .from(schema.contentDatabaseBodyHydrationQueue)
839
- .where(
840
- eq(
841
- schema.contentDatabaseBodyHydrationQueue.databaseItemId,
842
- args.databaseItemId,
843
- ),
900
+ const databaseItemIds = Array.from(
901
+ new Set(uniqueRequests.map((request) => request.databaseItemId)),
902
+ );
903
+ const existingRows: ContentDatabaseBodyHydrationQueueRowDb[] = [];
904
+ for (const idChunk of chunks(databaseItemIds, idChunkSize())) {
905
+ existingRows.push(
906
+ ...(await db
907
+ .select()
908
+ .from(schema.contentDatabaseBodyHydrationQueue)
909
+ .where(
910
+ inArray(
911
+ schema.contentDatabaseBodyHydrationQueue.databaseItemId,
912
+ idChunk,
913
+ ),
914
+ )),
844
915
  );
845
- const existingEntry = existing ? parseHydrationEntry(existing) : null;
846
- const shouldPreserveExistingEntry =
847
- builderEntryHasBodyContent(existingEntry) &&
848
- !builderEntryHasBodyContent(args.entry);
849
- const sourceEntryJson = shouldPreserveExistingEntry
850
- ? existing!.sourceEntryJson
851
- : JSON.stringify(args.entry);
852
- const values = {
853
- ownerEmail: args.ownerEmail,
854
- orgId: args.orgId,
855
- sourceId: args.sourceId,
856
- databaseItemId: args.databaseItemId,
857
- documentId: args.documentId,
858
- sourceRowId,
859
- sourceTable: args.sourceTable,
860
- sourceEntryJson,
861
- priority: Math.min(existing?.priority ?? priority, priority),
862
- lastError: null,
863
- updatedAt: args.now,
864
- };
865
- if (existing) {
866
- await db
867
- .update(schema.contentDatabaseBodyHydrationQueue)
868
- .set(values)
869
- .where(eq(schema.contentDatabaseBodyHydrationQueue.id, existing.id));
870
- } else {
871
- await db.insert(schema.contentDatabaseBodyHydrationQueue).values({
872
- id: crypto.randomUUID(),
873
- ...values,
874
- attempts: 0,
875
- lastAttemptedAt: null,
876
- createdAt: args.now,
916
+ }
917
+ const existingByItemId = new Map(
918
+ existingRows.map((row) => [row.databaseItemId, row]),
919
+ );
920
+ const queueRows: (typeof schema.contentDatabaseBodyHydrationQueue.$inferInsert)[] =
921
+ [];
922
+ for (const request of uniqueRequests) {
923
+ const existing = existingByItemId.get(request.databaseItemId);
924
+ const existingEntry = existing ? parseHydrationEntry(existing) : null;
925
+ const shouldPreserveExistingEntry =
926
+ builderEntryHasBodyContent(existingEntry) &&
927
+ !builderEntryHasBodyContent(request.entry);
928
+ const priority =
929
+ request.priority ??
930
+ builderBodyHydrationPriorityForRequest({ documentId: null });
931
+ queueRows.push({
932
+ id: existing?.id ?? crypto.randomUUID(),
933
+ ownerEmail: request.ownerEmail,
934
+ orgId: request.orgId,
935
+ sourceId: request.sourceId,
936
+ databaseItemId: request.databaseItemId,
937
+ documentId: request.documentId,
938
+ sourceRowId: request.entry.id,
939
+ sourceTable: request.sourceTable,
940
+ sourceEntryJson: shouldPreserveExistingEntry
941
+ ? existing!.sourceEntryJson
942
+ : JSON.stringify(request.entry),
943
+ priority: Math.min(existing?.priority ?? priority, priority),
944
+ attempts: existing?.attempts ?? 0,
945
+ lastAttemptedAt: existing?.lastAttemptedAt ?? null,
946
+ lastError: null,
947
+ createdAt: existing?.createdAt ?? request.now,
948
+ updatedAt: request.now,
877
949
  });
878
950
  }
879
- await db
880
- .update(schema.contentDatabaseItems)
881
- .set({
882
- bodyHydrationStatus: "pending",
883
- bodyHydrationError: null,
884
- updatedAt: args.now,
885
- })
886
- .where(eq(schema.contentDatabaseItems.id, args.databaseItemId));
951
+ const upsertedRows: ContentDatabaseBodyHydrationQueueRowDb[] = [];
952
+ for (const chunk of chunks(queueRows, bulkChunkSizeForColumnCount(15))) {
953
+ upsertedRows.push(
954
+ ...(await db
955
+ .insert(schema.contentDatabaseBodyHydrationQueue)
956
+ .values(chunk)
957
+ .onConflictDoUpdate({
958
+ target: schema.contentDatabaseBodyHydrationQueue.databaseItemId,
959
+ set: {
960
+ ownerEmail: sql`excluded.owner_email`,
961
+ orgId: sql`excluded.org_id`,
962
+ sourceId: sql`excluded.source_id`,
963
+ documentId: sql`excluded.document_id`,
964
+ sourceRowId: sql`excluded.source_row_id`,
965
+ sourceTable: sql`excluded.source_table`,
966
+ sourceEntryJson: sql`excluded.source_entry_json`,
967
+ priority: sql`excluded.priority`,
968
+ lastError: null,
969
+ updatedAt: sql`excluded.updated_at`,
970
+ },
971
+ })
972
+ .returning()),
973
+ );
974
+ }
975
+ for (const idChunk of chunks(databaseItemIds, idChunkSize())) {
976
+ await db
977
+ .update(schema.contentDatabaseItems)
978
+ .set({
979
+ bodyHydrationStatus: "pending",
980
+ bodyHydrationError: null,
981
+ updatedAt: uniqueRequests[0]!.now,
982
+ })
983
+ .where(inArray(schema.contentDatabaseItems.id, idChunk));
984
+ }
985
+ return upsertedRows;
887
986
  }
888
987
 
889
988
  export async function enqueueBuilderBodyHydrationForItems(args: {
@@ -896,9 +995,10 @@ export async function enqueueBuilderBodyHydrationForItems(args: {
896
995
  now: string;
897
996
  }) {
898
997
  if (!args.builderEntriesByDocumentId?.size) return;
998
+ const requests: BuilderBodyHydrationEnqueueRequest[] = [];
899
999
  for (const item of args.items) {
900
1000
  const entry = args.builderEntriesByDocumentId.get(item.document.id);
901
- if (!entry?.rawEntry) continue;
1001
+ if (!entry) continue;
902
1002
  if (
903
1003
  item.bodyHydration?.status === "hydrated" &&
904
1004
  item.bodyHydration.version === builderBodyHydrationVersion(entry) &&
@@ -907,7 +1007,7 @@ export async function enqueueBuilderBodyHydrationForItems(args: {
907
1007
  ) {
908
1008
  continue;
909
1009
  }
910
- await enqueueBuilderBodyHydration({
1010
+ requests.push({
911
1011
  sourceId: args.sourceId,
912
1012
  ownerEmail: args.ownerEmail,
913
1013
  orgId: args.orgId,
@@ -918,9 +1018,11 @@ export async function enqueueBuilderBodyHydrationForItems(args: {
918
1018
  now: args.now,
919
1019
  });
920
1020
  }
1021
+ const queuedJobs = await enqueueBuilderBodyHydrations(requests);
921
1022
  void processBuilderBodyHydrationQueue({
922
1023
  sourceId: args.sourceId,
923
1024
  limit: BUILDER_BODY_HYDRATION_BATCH_LIMIT,
1025
+ preloadedJobs: queuedJobs,
924
1026
  }).catch((error) => {
925
1027
  console.error("Builder body hydration background kick failed", error);
926
1028
  });
@@ -931,11 +1033,25 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
931
1033
  now: string;
932
1034
  }) {
933
1035
  const db = getDb();
1036
+ const requests: BuilderBodyHydrationEnqueueRequest[] = [];
934
1037
  const rows = await db
935
1038
  .select({
936
- item: schema.contentDatabaseItems,
937
- sourceRow: schema.contentDatabaseSourceRows,
938
- document: schema.documents,
1039
+ item: {
1040
+ id: schema.contentDatabaseItems.id,
1041
+ ownerEmail: schema.contentDatabaseItems.ownerEmail,
1042
+ orgId: schema.contentDatabaseItems.orgId,
1043
+ documentId: schema.contentDatabaseItems.documentId,
1044
+ },
1045
+ sourceRow: {
1046
+ sourceRowId: schema.contentDatabaseSourceRows.sourceRowId,
1047
+ sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
1048
+ lastSourceUpdatedAt:
1049
+ schema.contentDatabaseSourceRows.lastSourceUpdatedAt,
1050
+ },
1051
+ document: {
1052
+ title: schema.documents.title,
1053
+ content: schema.documents.content,
1054
+ },
939
1055
  })
940
1056
  .from(schema.contentDatabaseSourceRows)
941
1057
  .innerJoin(
@@ -949,14 +1065,21 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
949
1065
  schema.documents,
950
1066
  eq(schema.documents.id, schema.contentDatabaseSourceRows.documentId),
951
1067
  )
952
- .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
1068
+ .where(
1069
+ and(
1070
+ eq(schema.contentDatabaseSourceRows.sourceId, args.source.id),
1071
+ inArray(schema.contentDatabaseItems.bodyHydrationStatus, [
1072
+ "hydrated",
1073
+ "pending",
1074
+ ]),
1075
+ or(
1076
+ isNull(schema.documents.content),
1077
+ inArray(schema.documents.content, ["", "<empty-block/>"]),
1078
+ eq(sql<string>`TRIM(${schema.documents.content})`, ""),
1079
+ ),
1080
+ ),
1081
+ );
953
1082
  for (const row of rows) {
954
- if (
955
- row.item.bodyHydrationStatus !== "hydrated" &&
956
- row.item.bodyHydrationStatus !== "pending"
957
- )
958
- continue;
959
- if (!isEffectivelyEmptyDocumentContent(row.document.content)) continue;
960
1083
  const entry = builderEntryFromSourceRow({
961
1084
  row: row.sourceRow,
962
1085
  sourceTable: args.source.sourceTable,
@@ -965,9 +1088,8 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
965
1088
  const refreshedEntry = entry
966
1089
  ? await refreshBuilderBodySourceValuesFromStoredLossless(entry)
967
1090
  : null;
968
- if (!refreshedEntry || !builderEntryHasBodyContent(refreshedEntry))
969
- continue;
970
- await enqueueBuilderBodyHydration({
1091
+ if (!refreshedEntry) continue;
1092
+ requests.push({
971
1093
  sourceId: args.source.id,
972
1094
  ownerEmail: row.item.ownerEmail,
973
1095
  orgId: row.item.orgId,
@@ -978,6 +1100,15 @@ async function enqueueEmptyHydratedBuilderBodiesFromStoredRows(args: {
978
1100
  now: args.now,
979
1101
  });
980
1102
  }
1103
+ const queuedJobs = await enqueueBuilderBodyHydrations(requests);
1104
+ if (queuedJobs.length === 0) return;
1105
+ void processBuilderBodyHydrationQueue({
1106
+ sourceId: args.source.id,
1107
+ limit: BUILDER_BODY_HYDRATION_BATCH_LIMIT,
1108
+ preloadedJobs: queuedJobs,
1109
+ }).catch((error) => {
1110
+ console.error("Builder body hydration repair kick failed", error);
1111
+ });
981
1112
  }
982
1113
 
983
1114
  function parseHydrationEntry(
@@ -990,53 +1121,35 @@ function parseHydrationEntry(
990
1121
  async function processBuilderBodyHydrationJob(
991
1122
  row: ContentDatabaseBodyHydrationQueueRowDb,
992
1123
  now: string,
1124
+ preloaded?: {
1125
+ sourceRow?: ContentDatabaseSourceRecordRowDb | null;
1126
+ documentContent?: string | null;
1127
+ },
993
1128
  ) {
994
1129
  const db = getDb();
995
- const [claimed] = await db
996
- .update(schema.contentDatabaseBodyHydrationQueue)
997
- .set({
998
- attempts: row.attempts + 1,
999
- lastAttemptedAt: now,
1000
- lastError: null,
1001
- updatedAt: now,
1002
- })
1003
- .where(
1004
- and(
1005
- eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
1006
- eq(
1007
- schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1008
- row.sourceEntryJson,
1009
- ),
1010
- ),
1011
- )
1012
- .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1013
- if (!claimed) return;
1014
-
1015
- await db
1016
- .update(schema.contentDatabaseItems)
1017
- .set({
1018
- bodyHydrationStatus: "hydrating",
1019
- bodyHydrationAttemptedAt: now,
1020
- bodyHydrationError: null,
1021
- updatedAt: now,
1022
- })
1023
- .where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
1024
-
1025
1130
  const entry = parseHydrationEntry(row);
1026
1131
  if (!entry) throw new Error("Builder body hydration entry is missing.");
1027
1132
  let activeSourceEntryJson = row.sourceEntryJson;
1028
1133
  let entryWithBody = await refreshBuilderBodySourceValuesFromStoredLossless(
1029
1134
  await withBuilderBodySourceValues(entry),
1030
1135
  );
1031
- const [sourceRow] = await db
1032
- .select()
1033
- .from(schema.contentDatabaseSourceRows)
1034
- .where(
1035
- and(
1036
- eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
1037
- eq(schema.contentDatabaseSourceRows.databaseItemId, row.databaseItemId),
1038
- ),
1039
- );
1136
+ const sourceRow =
1137
+ preloaded?.sourceRow != null
1138
+ ? (preloaded.sourceRow ?? undefined)
1139
+ : (
1140
+ await db
1141
+ .select()
1142
+ .from(schema.contentDatabaseSourceRows)
1143
+ .where(
1144
+ and(
1145
+ eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
1146
+ eq(
1147
+ schema.contentDatabaseSourceRows.databaseItemId,
1148
+ row.databaseItemId,
1149
+ ),
1150
+ ),
1151
+ )
1152
+ )[0];
1040
1153
  const sourceValues =
1041
1154
  parseObject<Record<string, DocumentPropertyValue>>(
1042
1155
  sourceRow?.sourceValuesJson ?? "{}",
@@ -1132,20 +1245,40 @@ async function processBuilderBodyHydrationJob(
1132
1245
  }
1133
1246
  }
1134
1247
  if (!nextContent.trim()) {
1135
- const attempts = row.attempts + 1;
1248
+ const attempts = row.attempts;
1136
1249
  await db.transaction(async (tx) => {
1137
- if (builderBodyHydrationAttemptIsTerminal(attempts)) {
1250
+ const queueRowCas = and(
1251
+ eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
1252
+ eq(
1253
+ schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1254
+ activeSourceEntryJson,
1255
+ ),
1256
+ );
1257
+ const markPendingIfReplaced = async () => {
1258
+ const [replacedByNewerJob] = await tx
1259
+ .select({ id: schema.contentDatabaseBodyHydrationQueue.id })
1260
+ .from(schema.contentDatabaseBodyHydrationQueue)
1261
+ .where(eq(schema.contentDatabaseBodyHydrationQueue.id, row.id));
1262
+ if (!replacedByNewerJob) return;
1138
1263
  await tx
1264
+ .update(schema.contentDatabaseItems)
1265
+ .set({
1266
+ bodyHydrationStatus: "pending",
1267
+ bodyHydrationAttemptedAt: now,
1268
+ bodyHydrationError: null,
1269
+ updatedAt: now,
1270
+ })
1271
+ .where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
1272
+ };
1273
+ if (builderBodyHydrationAttemptIsTerminal(attempts)) {
1274
+ const [deleted] = await tx
1139
1275
  .delete(schema.contentDatabaseBodyHydrationQueue)
1140
- .where(
1141
- and(
1142
- eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
1143
- eq(
1144
- schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1145
- activeSourceEntryJson,
1146
- ),
1147
- ),
1148
- );
1276
+ .where(queueRowCas)
1277
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1278
+ if (!deleted) {
1279
+ await markPendingIfReplaced();
1280
+ return;
1281
+ }
1149
1282
  await tx
1150
1283
  .update(schema.contentDatabaseItems)
1151
1284
  .set({
@@ -1157,6 +1290,18 @@ async function processBuilderBodyHydrationJob(
1157
1290
  .where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
1158
1291
  return;
1159
1292
  }
1293
+ const [stillOwnsQueueRow] = await tx
1294
+ .update(schema.contentDatabaseBodyHydrationQueue)
1295
+ .set({
1296
+ lastError: BUILDER_BODY_NOT_AVAILABLE_ERROR,
1297
+ updatedAt: now,
1298
+ })
1299
+ .where(queueRowCas)
1300
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1301
+ if (!stillOwnsQueueRow) {
1302
+ await markPendingIfReplaced();
1303
+ return;
1304
+ }
1160
1305
  await tx
1161
1306
  .update(schema.contentDatabaseItems)
1162
1307
  .set({
@@ -1170,13 +1315,18 @@ async function processBuilderBodyHydrationJob(
1170
1315
  return;
1171
1316
  }
1172
1317
  }
1173
- const [document] = await db
1174
- .select({ content: schema.documents.content })
1175
- .from(schema.documents)
1176
- .where(eq(schema.documents.id, row.documentId));
1318
+ const documentContent =
1319
+ preloaded?.documentContent != null
1320
+ ? preloaded.documentContent
1321
+ : (
1322
+ await db
1323
+ .select({ content: schema.documents.content })
1324
+ .from(schema.documents)
1325
+ .where(eq(schema.documents.id, row.documentId))
1326
+ )[0]?.content;
1177
1327
  const previousContent =
1178
1328
  stringSourceValue(sourceValues, BUILDER_CMS_BODY_CONTENT_KEY) ?? "";
1179
- const currentContent = document?.content ?? "";
1329
+ const currentContent = documentContent ?? "";
1180
1330
  const shouldWriteBody =
1181
1331
  currentContent === "" ||
1182
1332
  isEffectivelyEmptyDocumentContent(currentContent) ||
@@ -1190,6 +1340,40 @@ async function processBuilderBodyHydrationJob(
1190
1340
  });
1191
1341
  let wroteBody = false;
1192
1342
  await db.transaction(async (tx) => {
1343
+ const queueRowCas = and(
1344
+ eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
1345
+ eq(
1346
+ schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1347
+ activeSourceEntryJson,
1348
+ ),
1349
+ );
1350
+ const markPendingIfReplaced = async () => {
1351
+ const [replacedByNewerJob] = await tx
1352
+ .select({ id: schema.contentDatabaseBodyHydrationQueue.id })
1353
+ .from(schema.contentDatabaseBodyHydrationQueue)
1354
+ .where(eq(schema.contentDatabaseBodyHydrationQueue.id, row.id));
1355
+ if (!replacedByNewerJob) return;
1356
+ await tx
1357
+ .update(schema.contentDatabaseItems)
1358
+ .set({
1359
+ bodyHydrationStatus: "pending",
1360
+ bodyHydrationAttemptedAt: now,
1361
+ bodyHydrationError: null,
1362
+ updatedAt: now,
1363
+ })
1364
+ .where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
1365
+ };
1366
+ const [stillOwnsQueueRow] = await tx
1367
+ .update(schema.contentDatabaseBodyHydrationQueue)
1368
+ .set({
1369
+ updatedAt: now,
1370
+ })
1371
+ .where(queueRowCas)
1372
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1373
+ if (!stillOwnsQueueRow) {
1374
+ await markPendingIfReplaced();
1375
+ return;
1376
+ }
1193
1377
  if (shouldWriteBody) {
1194
1378
  const contentCas =
1195
1379
  isEffectivelyEmptyDocumentContent(currentContent) &&
@@ -1215,15 +1399,7 @@ async function processBuilderBodyHydrationJob(
1215
1399
  "Skipped Builder body hydration because the document changed during sync.",
1216
1400
  updatedAt: now,
1217
1401
  })
1218
- .where(
1219
- and(
1220
- eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
1221
- eq(
1222
- schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1223
- activeSourceEntryJson,
1224
- ),
1225
- ),
1226
- )
1402
+ .where(queueRowCas)
1227
1403
  .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1228
1404
  if (stillQueued) {
1229
1405
  await tx
@@ -1239,38 +1415,55 @@ async function processBuilderBodyHydrationJob(
1239
1415
  }
1240
1416
  return;
1241
1417
  }
1242
- const [deleted] = await tx
1243
- .delete(schema.contentDatabaseBodyHydrationQueue)
1244
- .where(
1245
- and(
1246
- eq(schema.contentDatabaseBodyHydrationQueue.id, row.id),
1247
- eq(
1248
- schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1249
- activeSourceEntryJson,
1250
- ),
1251
- ),
1252
- )
1253
- .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1254
- if (!deleted) {
1255
- if (wroteBody) {
1418
+ const sourceRowWhere = and(
1419
+ eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
1420
+ eq(schema.contentDatabaseSourceRows.databaseItemId, row.databaseItemId),
1421
+ sourceRow
1422
+ ? eq(
1423
+ schema.contentDatabaseSourceRows.sourceValuesJson,
1424
+ sourceRow.sourceValuesJson,
1425
+ )
1426
+ : isNull(schema.contentDatabaseSourceRows.id),
1427
+ );
1428
+ const [updatedSourceRow] = await tx
1429
+ .update(schema.contentDatabaseSourceRows)
1430
+ .set({
1431
+ sourceValuesJson: JSON.stringify(nextValues),
1432
+ lastSyncedAt: now,
1433
+ lastSourceUpdatedAt: entryWithBody.updatedAt ?? now,
1434
+ updatedAt: now,
1435
+ })
1436
+ .where(sourceRowWhere)
1437
+ .returning({ id: schema.contentDatabaseSourceRows.id });
1438
+ if (!updatedSourceRow) {
1439
+ const [stillQueued] = await tx
1440
+ .update(schema.contentDatabaseBodyHydrationQueue)
1441
+ .set({
1442
+ lastError:
1443
+ "Skipped Builder body hydration because the source row changed during sync.",
1444
+ updatedAt: now,
1445
+ })
1446
+ .where(queueRowCas)
1447
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1448
+ if (stillQueued) {
1256
1449
  await tx
1257
- .update(schema.contentDatabaseSourceRows)
1450
+ .update(schema.contentDatabaseItems)
1258
1451
  .set({
1259
- sourceValuesJson: JSON.stringify(nextValues),
1260
- lastSyncedAt: now,
1261
- lastSourceUpdatedAt: entryWithBody.updatedAt ?? now,
1452
+ bodyHydrationStatus: "pending",
1453
+ bodyHydrationAttemptedAt: now,
1454
+ bodyHydrationError:
1455
+ "Skipped Builder body hydration because the source row changed during sync.",
1262
1456
  updatedAt: now,
1263
1457
  })
1264
- .where(
1265
- and(
1266
- eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
1267
- eq(
1268
- schema.contentDatabaseSourceRows.databaseItemId,
1269
- row.databaseItemId,
1270
- ),
1271
- ),
1272
- );
1458
+ .where(eq(schema.contentDatabaseItems.id, row.databaseItemId));
1273
1459
  }
1460
+ return;
1461
+ }
1462
+ const [deleted] = await tx
1463
+ .delete(schema.contentDatabaseBodyHydrationQueue)
1464
+ .where(queueRowCas)
1465
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1466
+ if (!deleted) {
1274
1467
  // Our delete matched nothing: either a NEWER job version replaced this
1275
1468
  // row (same id, different sourceEntryJson — mark pending so the newer
1276
1469
  // job's processor owns it), or a concurrent processor completed and
@@ -1293,23 +1486,6 @@ async function processBuilderBodyHydrationJob(
1293
1486
  }
1294
1487
  return;
1295
1488
  }
1296
- await tx
1297
- .update(schema.contentDatabaseSourceRows)
1298
- .set({
1299
- sourceValuesJson: JSON.stringify(nextValues),
1300
- lastSyncedAt: now,
1301
- lastSourceUpdatedAt: entryWithBody.updatedAt ?? now,
1302
- updatedAt: now,
1303
- })
1304
- .where(
1305
- and(
1306
- eq(schema.contentDatabaseSourceRows.sourceId, row.sourceId),
1307
- eq(
1308
- schema.contentDatabaseSourceRows.databaseItemId,
1309
- row.databaseItemId,
1310
- ),
1311
- ),
1312
- );
1313
1489
  await tx
1314
1490
  .update(schema.contentDatabaseItems)
1315
1491
  .set({
@@ -1408,6 +1584,7 @@ export async function processBuilderBodyHydrationQueue(args: {
1408
1584
  sourceId: string;
1409
1585
  documentId?: string | null;
1410
1586
  limit?: number | null;
1587
+ preloadedJobs?: ContentDatabaseBodyHydrationQueueRowDb[];
1411
1588
  }) {
1412
1589
  const db = getDb();
1413
1590
  const limit = normalizeHydrationLimit(args.limit);
@@ -1434,69 +1611,229 @@ export async function processBuilderBodyHydrationQueue(args: {
1434
1611
  ),
1435
1612
  );
1436
1613
  }
1437
- const jobs = await db
1438
- .select()
1439
- .from(schema.contentDatabaseBodyHydrationQueue)
1440
- .where(
1441
- args.documentId
1442
- ? and(
1443
- eq(
1614
+ const persistedJobs = async (queryLimit: number) =>
1615
+ db
1616
+ .select()
1617
+ .from(schema.contentDatabaseBodyHydrationQueue)
1618
+ .where(
1619
+ args.documentId
1620
+ ? and(
1621
+ eq(
1622
+ schema.contentDatabaseBodyHydrationQueue.sourceId,
1623
+ args.sourceId,
1624
+ ),
1625
+ eq(
1626
+ schema.contentDatabaseBodyHydrationQueue.documentId,
1627
+ args.documentId,
1628
+ ),
1629
+ )
1630
+ : eq(
1444
1631
  schema.contentDatabaseBodyHydrationQueue.sourceId,
1445
1632
  args.sourceId,
1446
1633
  ),
1447
- eq(
1448
- schema.contentDatabaseBodyHydrationQueue.documentId,
1449
- args.documentId,
1450
- ),
1451
- )
1452
- : eq(schema.contentDatabaseBodyHydrationQueue.sourceId, args.sourceId),
1453
- )
1454
- .orderBy(
1455
- asc(schema.contentDatabaseBodyHydrationQueue.priority),
1456
- asc(schema.contentDatabaseBodyHydrationQueue.createdAt),
1457
- )
1458
- .limit(limit);
1634
+ )
1635
+ .orderBy(
1636
+ asc(schema.contentDatabaseBodyHydrationQueue.priority),
1637
+ asc(schema.contentDatabaseBodyHydrationQueue.createdAt),
1638
+ )
1639
+ .limit(queryLimit);
1640
+ const jobs = await (args.preloadedJobs?.length && !args.documentId
1641
+ ? (() => {
1642
+ const preloadedJobs = sortBuilderBodyHydrationQueueForProcessing(
1643
+ args.preloadedJobs!.filter((job) => job.sourceId === args.sourceId),
1644
+ ).slice(0, limit);
1645
+ return persistedJobs(limit + preloadedJobs.length).then((rows) => {
1646
+ const preloadedIds = new Set(preloadedJobs.map((job) => job.id));
1647
+ return sortBuilderBodyHydrationQueueForProcessing([
1648
+ ...preloadedJobs,
1649
+ ...rows.filter((row) => !preloadedIds.has(row.id)),
1650
+ ]).slice(0, limit);
1651
+ });
1652
+ })()
1653
+ : persistedJobs(limit));
1459
1654
 
1460
1655
  let succeeded = 0;
1461
1656
  let failed = 0;
1462
- for (const job of jobs) {
1463
- const attemptNow = new Date().toISOString();
1464
- try {
1465
- const delayMs = builderBodyHydrationDelayMs();
1466
- if (delayMs > 0) await sleep(delayMs);
1467
- await processBuilderBodyHydrationJob(job, attemptNow);
1468
- succeeded += 1;
1469
- } catch (error) {
1470
- failed += 1;
1471
- const message = error instanceof Error ? error.message : String(error);
1472
- const attempts = job.attempts + 1;
1473
- await db
1474
- .update(schema.contentDatabaseItems)
1475
- .set({
1476
- bodyHydrationStatus: "error",
1477
- bodyHydrationAttemptedAt: attemptNow,
1478
- bodyHydrationError: message,
1479
- updatedAt: attemptNow,
1480
- })
1481
- .where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
1482
- if (builderBodyHydrationAttemptIsTerminal(attempts)) {
1483
- await db
1484
- .delete(schema.contentDatabaseBodyHydrationQueue)
1485
- .where(eq(schema.contentDatabaseBodyHydrationQueue.id, job.id));
1486
- continue;
1487
- }
1488
- await db
1657
+ const claimedJobs: ContentDatabaseBodyHydrationQueueRowDb[] = [];
1658
+ for (const jobChunk of chunks(jobs, bulkChunkSizeForColumnCount(2))) {
1659
+ const claimFilters = jobChunk.map((job) =>
1660
+ and(
1661
+ eq(schema.contentDatabaseBodyHydrationQueue.id, job.id),
1662
+ eq(
1663
+ schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1664
+ job.sourceEntryJson,
1665
+ ),
1666
+ ),
1667
+ );
1668
+ if (claimFilters.length === 0) continue;
1669
+ claimedJobs.push(
1670
+ ...(await db
1489
1671
  .update(schema.contentDatabaseBodyHydrationQueue)
1490
1672
  .set({
1491
- attempts,
1492
- lastAttemptedAt: attemptNow,
1493
- lastError: message,
1494
- priority: job.priority + 10,
1495
- updatedAt: attemptNow,
1673
+ attempts: sql`${schema.contentDatabaseBodyHydrationQueue.attempts} + 1`,
1674
+ lastAttemptedAt: now,
1675
+ lastError: null,
1676
+ updatedAt: now,
1496
1677
  })
1497
- .where(eq(schema.contentDatabaseBodyHydrationQueue.id, job.id));
1498
- }
1678
+ .where(or(...claimFilters))
1679
+ .returning()),
1680
+ );
1681
+ }
1682
+ for (const idChunk of chunks(
1683
+ claimedJobs.map((job) => job.databaseItemId),
1684
+ idChunkSize(),
1685
+ )) {
1686
+ await db
1687
+ .update(schema.contentDatabaseItems)
1688
+ .set({
1689
+ bodyHydrationStatus: "hydrating",
1690
+ bodyHydrationAttemptedAt: now,
1691
+ bodyHydrationError: null,
1692
+ updatedAt: now,
1693
+ })
1694
+ .where(inArray(schema.contentDatabaseItems.id, idChunk));
1499
1695
  }
1696
+ const sourceRows =
1697
+ claimedJobs.length > 0
1698
+ ? (
1699
+ await Promise.all(
1700
+ chunks(
1701
+ claimedJobs.map((job) => job.databaseItemId),
1702
+ idChunkSize(),
1703
+ ).map((idChunk) =>
1704
+ db
1705
+ .select()
1706
+ .from(schema.contentDatabaseSourceRows)
1707
+ .where(
1708
+ and(
1709
+ eq(
1710
+ schema.contentDatabaseSourceRows.sourceId,
1711
+ args.sourceId,
1712
+ ),
1713
+ inArray(
1714
+ schema.contentDatabaseSourceRows.databaseItemId,
1715
+ idChunk,
1716
+ ),
1717
+ ),
1718
+ ),
1719
+ ),
1720
+ )
1721
+ ).flat()
1722
+ : [];
1723
+ const sourceRowsByItemId = new Map(
1724
+ sourceRows.map((row) => [row.databaseItemId, row]),
1725
+ );
1726
+ const documents =
1727
+ claimedJobs.length > 0
1728
+ ? (
1729
+ await Promise.all(
1730
+ chunks(
1731
+ Array.from(new Set(claimedJobs.map((job) => job.documentId))),
1732
+ idChunkSize(),
1733
+ ).map((idChunk) =>
1734
+ db
1735
+ .select({
1736
+ id: schema.documents.id,
1737
+ content: schema.documents.content,
1738
+ })
1739
+ .from(schema.documents)
1740
+ .where(inArray(schema.documents.id, idChunk)),
1741
+ ),
1742
+ )
1743
+ ).flat()
1744
+ : [];
1745
+ const documentContentById = new Map(
1746
+ documents.map((document) => [document.id, document.content]),
1747
+ );
1748
+ await processInBatches(
1749
+ claimedJobs,
1750
+ BUILDER_BODY_HYDRATION_PROCESS_CONCURRENCY,
1751
+ async (job) => {
1752
+ const attemptNow = new Date().toISOString();
1753
+ try {
1754
+ const delayMs = builderBodyHydrationDelayMs();
1755
+ if (delayMs > 0) await sleep(delayMs);
1756
+ await processBuilderBodyHydrationJob(job, attemptNow, {
1757
+ sourceRow: sourceRowsByItemId.get(job.databaseItemId) ?? null,
1758
+ documentContent: documentContentById.has(job.documentId)
1759
+ ? (documentContentById.get(job.documentId) ?? null)
1760
+ : undefined,
1761
+ });
1762
+ succeeded += 1;
1763
+ } catch (error) {
1764
+ failed += 1;
1765
+ const message = error instanceof Error ? error.message : String(error);
1766
+ const attempts = job.attempts;
1767
+ const queueRowCas = and(
1768
+ eq(schema.contentDatabaseBodyHydrationQueue.id, job.id),
1769
+ eq(
1770
+ schema.contentDatabaseBodyHydrationQueue.sourceEntryJson,
1771
+ job.sourceEntryJson,
1772
+ ),
1773
+ );
1774
+ const markPendingIfReplaced = async () => {
1775
+ const [replacedByNewerJob] = await db
1776
+ .select({ id: schema.contentDatabaseBodyHydrationQueue.id })
1777
+ .from(schema.contentDatabaseBodyHydrationQueue)
1778
+ .where(eq(schema.contentDatabaseBodyHydrationQueue.id, job.id));
1779
+ if (!replacedByNewerJob) return;
1780
+ await db
1781
+ .update(schema.contentDatabaseItems)
1782
+ .set({
1783
+ bodyHydrationStatus: "pending",
1784
+ bodyHydrationAttemptedAt: attemptNow,
1785
+ bodyHydrationError: null,
1786
+ updatedAt: attemptNow,
1787
+ })
1788
+ .where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
1789
+ };
1790
+ if (builderBodyHydrationAttemptIsTerminal(attempts)) {
1791
+ const [deleted] = await db
1792
+ .delete(schema.contentDatabaseBodyHydrationQueue)
1793
+ .where(queueRowCas)
1794
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1795
+ if (!deleted) {
1796
+ await markPendingIfReplaced();
1797
+ return;
1798
+ }
1799
+ await db
1800
+ .update(schema.contentDatabaseItems)
1801
+ .set({
1802
+ bodyHydrationStatus: "error",
1803
+ bodyHydrationAttemptedAt: attemptNow,
1804
+ bodyHydrationError: message,
1805
+ updatedAt: attemptNow,
1806
+ })
1807
+ .where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
1808
+ return;
1809
+ }
1810
+ const [updatedQueueRow] = await db
1811
+ .update(schema.contentDatabaseBodyHydrationQueue)
1812
+ .set({
1813
+ attempts,
1814
+ lastAttemptedAt: attemptNow,
1815
+ lastError: message,
1816
+ priority: job.priority + 10,
1817
+ updatedAt: attemptNow,
1818
+ })
1819
+ .where(queueRowCas)
1820
+ .returning({ id: schema.contentDatabaseBodyHydrationQueue.id });
1821
+ if (!updatedQueueRow) {
1822
+ await markPendingIfReplaced();
1823
+ return;
1824
+ }
1825
+ await db
1826
+ .update(schema.contentDatabaseItems)
1827
+ .set({
1828
+ bodyHydrationStatus: "error",
1829
+ bodyHydrationAttemptedAt: attemptNow,
1830
+ bodyHydrationError: message,
1831
+ updatedAt: attemptNow,
1832
+ })
1833
+ .where(eq(schema.contentDatabaseItems.id, job.databaseItemId));
1834
+ }
1835
+ },
1836
+ );
1500
1837
  const [remaining] = await db
1501
1838
  .select({ count: sql<number>`COUNT(*)` })
1502
1839
  .from(schema.contentDatabaseBodyHydrationQueue)
@@ -1505,7 +1842,7 @@ export async function processBuilderBodyHydrationQueue(args: {
1505
1842
  );
1506
1843
  return {
1507
1844
  sourceId: args.sourceId,
1508
- processed: jobs.length,
1845
+ processed: claimedJobs.length,
1509
1846
  succeeded,
1510
1847
  failed,
1511
1848
  remaining: Number(remaining?.count ?? 0),
@@ -3160,7 +3497,7 @@ function builderSourceValuesWithPreservedBodyBaseline(args: {
3160
3497
  return next;
3161
3498
  }
3162
3499
 
3163
- async function materializeSourceFieldPropertyValues(args: {
3500
+ export async function materializeSourceFieldPropertyValues(args: {
3164
3501
  database: ContentDatabaseRow;
3165
3502
  sourceId: string;
3166
3503
  fields: ContentDatabaseSourceFieldRowDb[];
@@ -3184,25 +3521,104 @@ async function materializeSourceFieldPropertyValues(args: {
3184
3521
  const definitionById = new Map(
3185
3522
  definitions.map((definition) => [definition.id, definition]),
3186
3523
  );
3187
- const sourceRows = await db
3188
- .select()
3189
- .from(schema.contentDatabaseSourceRows)
3190
- .where(eq(schema.contentDatabaseSourceRows.sourceId, args.sourceId));
3191
- const scopedRows = documentIdSet
3192
- ? sourceRows.filter((row) => documentIdSet.has(row.documentId))
3193
- : sourceRows;
3524
+ const scopedRows: Array<{
3525
+ documentId: string;
3526
+ sourceValuesJson: string;
3527
+ }> = [];
3528
+ if (documentIdSet) {
3529
+ for (const idChunk of chunks(Array.from(documentIdSet), idChunkSize())) {
3530
+ scopedRows.push(
3531
+ ...(await db
3532
+ .select({
3533
+ documentId: schema.contentDatabaseSourceRows.documentId,
3534
+ sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
3535
+ })
3536
+ .from(schema.contentDatabaseSourceRows)
3537
+ .where(
3538
+ and(
3539
+ eq(schema.contentDatabaseSourceRows.sourceId, args.sourceId),
3540
+ inArray(schema.contentDatabaseSourceRows.documentId, idChunk),
3541
+ ),
3542
+ )
3543
+ .orderBy(
3544
+ asc(schema.contentDatabaseSourceRows.updatedAt),
3545
+ asc(schema.contentDatabaseSourceRows.createdAt),
3546
+ asc(schema.contentDatabaseSourceRows.id),
3547
+ )),
3548
+ );
3549
+ }
3550
+ } else {
3551
+ scopedRows.push(
3552
+ ...(await db
3553
+ .select({
3554
+ documentId: schema.contentDatabaseSourceRows.documentId,
3555
+ sourceValuesJson: schema.contentDatabaseSourceRows.sourceValuesJson,
3556
+ })
3557
+ .from(schema.contentDatabaseSourceRows)
3558
+ .where(eq(schema.contentDatabaseSourceRows.sourceId, args.sourceId))
3559
+ .orderBy(
3560
+ asc(schema.contentDatabaseSourceRows.updatedAt),
3561
+ asc(schema.contentDatabaseSourceRows.createdAt),
3562
+ asc(schema.contentDatabaseSourceRows.id),
3563
+ )),
3564
+ );
3565
+ }
3194
3566
  if (scopedRows.length === 0) return;
3195
3567
 
3196
- const existingValues = await db
3197
- .select()
3198
- .from(schema.documentPropertyValues)
3199
- .where(inArray(schema.documentPropertyValues.propertyId, propertyIds));
3568
+ const existingValues: Array<
3569
+ typeof schema.documentPropertyValues.$inferSelect
3570
+ > = [];
3571
+ const scopedExistingValueChunkSize = documentIdSet
3572
+ ? bulkChunkSizeForColumnCount(2)
3573
+ : idChunkSize();
3574
+ for (const propertyIdChunk of chunks(
3575
+ propertyIds,
3576
+ scopedExistingValueChunkSize,
3577
+ )) {
3578
+ if (documentIdSet) {
3579
+ for (const documentIdChunk of chunks(
3580
+ Array.from(documentIdSet),
3581
+ scopedExistingValueChunkSize,
3582
+ )) {
3583
+ existingValues.push(
3584
+ ...(await db
3585
+ .select()
3586
+ .from(schema.documentPropertyValues)
3587
+ .where(
3588
+ and(
3589
+ inArray(
3590
+ schema.documentPropertyValues.propertyId,
3591
+ propertyIdChunk,
3592
+ ),
3593
+ inArray(
3594
+ schema.documentPropertyValues.documentId,
3595
+ documentIdChunk,
3596
+ ),
3597
+ ),
3598
+ )),
3599
+ );
3600
+ }
3601
+ } else {
3602
+ existingValues.push(
3603
+ ...(await db
3604
+ .select()
3605
+ .from(schema.documentPropertyValues)
3606
+ .where(
3607
+ inArray(schema.documentPropertyValues.propertyId, propertyIdChunk),
3608
+ )),
3609
+ );
3610
+ }
3611
+ }
3200
3612
  const existingByDocumentAndProperty = new Map(
3201
3613
  existingValues.map((value) => [
3202
3614
  `${value.documentId}\0${value.propertyId}`,
3203
3615
  value,
3204
3616
  ]),
3205
3617
  );
3618
+ const upsertsByDocumentAndProperty = new Map<
3619
+ string,
3620
+ typeof schema.documentPropertyValues.$inferInsert
3621
+ >();
3206
3622
 
3207
3623
  for (const row of scopedRows) {
3208
3624
  const sourceValues =
@@ -3222,27 +3638,34 @@ async function materializeSourceFieldPropertyValues(args: {
3222
3638
  const valueJson = serializePropertyValue(normalized);
3223
3639
  const key = `${row.documentId}\0${propertyId}`;
3224
3640
  const existing = existingByDocumentAndProperty.get(key);
3225
- if (existing) {
3226
- if (existing.valueJson === valueJson) continue;
3227
- await db
3228
- .update(schema.documentPropertyValues)
3229
- .set({ valueJson, updatedAt: args.now })
3230
- .where(eq(schema.documentPropertyValues.id, existing.id));
3231
- } else {
3232
- const value = {
3233
- id: nanoid(),
3234
- ownerEmail: args.database.ownerEmail,
3235
- documentId: row.documentId,
3236
- propertyId,
3237
- valueJson,
3238
- createdAt: args.now,
3239
- updatedAt: args.now,
3240
- };
3241
- await db.insert(schema.documentPropertyValues).values(value);
3242
- existingByDocumentAndProperty.set(key, value);
3243
- }
3641
+ if (existing?.valueJson === valueJson) continue;
3642
+ upsertsByDocumentAndProperty.set(key, {
3643
+ id: existing?.id ?? nanoid(),
3644
+ ownerEmail: existing?.ownerEmail ?? args.database.ownerEmail,
3645
+ documentId: row.documentId,
3646
+ propertyId,
3647
+ valueJson,
3648
+ createdAt: existing?.createdAt ?? args.now,
3649
+ updatedAt: args.now,
3650
+ });
3244
3651
  }
3245
3652
  }
3653
+ for (const chunk of chunks(
3654
+ Array.from(upsertsByDocumentAndProperty.values()),
3655
+ bulkChunkSizeForColumnCount(7),
3656
+ )) {
3657
+ if (chunk.length === 0) continue;
3658
+ await db
3659
+ .insert(schema.documentPropertyValues)
3660
+ .values(chunk)
3661
+ .onConflictDoUpdate({
3662
+ target: schema.documentPropertyValues.id,
3663
+ set: {
3664
+ valueJson: sql`excluded.value_json`,
3665
+ updatedAt: sql`excluded.updated_at`,
3666
+ },
3667
+ });
3668
+ }
3246
3669
  }
3247
3670
 
3248
3671
  function openChangeSetKey(row: ContentDatabaseSourceChangeSetRowDb) {
@@ -3643,7 +4066,8 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
3643
4066
 
3644
4067
  let nextDocPosition = (maxDocPos?.max ?? -1) + 1;
3645
4068
  let nextItemPosition = (maxItemPos?.max ?? -1) + 1;
3646
- let imported = 0;
4069
+ const documentRows: (typeof schema.documents.$inferInsert)[] = [];
4070
+ const itemRows: (typeof schema.contentDatabaseItems.$inferInsert)[] = [];
3647
4071
 
3648
4072
  for (const entry of args.entries) {
3649
4073
  if (
@@ -3663,7 +4087,7 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
3663
4087
 
3664
4088
  const documentId = nanoid();
3665
4089
  const itemId = nanoid();
3666
- await db.insert(schema.documents).values({
4090
+ documentRows.push({
3667
4091
  id: documentId,
3668
4092
  ownerEmail: args.database.ownerEmail,
3669
4093
  orgId: args.database.orgId,
@@ -3678,23 +4102,30 @@ export async function importBuilderCmsEntriesAsDatabaseItems(args: {
3678
4102
  createdAt: args.now,
3679
4103
  updatedAt: args.now,
3680
4104
  });
3681
- await db.insert(schema.contentDatabaseItems).values({
4105
+ itemRows.push({
3682
4106
  id: itemId,
3683
4107
  ownerEmail: args.database.ownerEmail,
3684
4108
  orgId: args.database.orgId,
3685
4109
  databaseId: args.database.id,
3686
4110
  documentId,
3687
4111
  position: nextItemPosition++,
3688
- bodyHydrationStatus: entry.rawEntry ? "pending" : "hydrated",
4112
+ bodyHydrationStatus: "pending",
3689
4113
  bodyHydrationError: null,
3690
4114
  createdAt: args.now,
3691
4115
  updatedAt: args.now,
3692
4116
  });
3693
- imported += 1;
3694
4117
  importedEntriesByDocumentId.set(documentId, entry);
3695
4118
  }
4119
+ await db.transaction(async (tx) => {
4120
+ for (const chunk of chunks(documentRows, bulkChunkSizeForColumnCount(13))) {
4121
+ await tx.insert(schema.documents).values(chunk);
4122
+ }
4123
+ for (const chunk of chunks(itemRows, bulkChunkSizeForColumnCount(10))) {
4124
+ await tx.insert(schema.contentDatabaseItems).values(chunk);
4125
+ }
4126
+ });
3696
4127
 
3697
- return { imported, importedEntriesByDocumentId };
4128
+ return { imported: itemRows.length, importedEntriesByDocumentId };
3698
4129
  }
3699
4130
 
3700
4131
  export async function resyncBuilderCmsSourceSnapshot(args: {
@@ -3765,9 +4196,19 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
3765
4196
  .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
3766
4197
  }
3767
4198
  }
3768
- const builderModelFields = await readBuilderCmsModelFields({
3769
- model: args.source.sourceTable,
3770
- });
4199
+ let builderModelFields: BuilderCmsModelFieldSummary[] | undefined;
4200
+ let builderModelFieldsReadFailed = false;
4201
+ try {
4202
+ builderModelFields = await readBuilderCmsModelFields({
4203
+ model: args.source.sourceTable,
4204
+ });
4205
+ } catch (error) {
4206
+ builderModelFieldsReadFailed = true;
4207
+ const message = error instanceof Error ? error.message : String(error);
4208
+ console.warn(
4209
+ `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`,
4210
+ );
4211
+ }
3771
4212
  const builderEntriesByDocumentId =
3772
4213
  builderRead.state === "live"
3773
4214
  ? mapBuilderCmsEntriesToLocalItems({
@@ -3806,36 +4247,39 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
3806
4247
  );
3807
4248
  const hasMore = builderRead.progress?.hasMore === true;
3808
4249
 
3809
- await db
3810
- .delete(schema.contentDatabaseSourceFields)
3811
- .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
3812
- await seedMockSourceFields({
3813
- sourceId: args.source.id,
3814
- ownerEmail: args.database.ownerEmail,
3815
- sourceType: "builder-cms",
3816
- properties,
3817
- builderModelFields,
3818
- builderSampleEntries: builderEntries,
3819
- existingFields,
3820
- now: args.now,
3821
- });
4250
+ const shouldReseedSourceFields =
4251
+ !builderModelFieldsReadFailed || existingFields.length === 0;
4252
+ if (shouldReseedSourceFields) {
4253
+ await db
4254
+ .delete(schema.contentDatabaseSourceFields)
4255
+ .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
4256
+ await seedMockSourceFields({
4257
+ sourceId: args.source.id,
4258
+ ownerEmail: args.database.ownerEmail,
4259
+ sourceType: "builder-cms",
4260
+ properties,
4261
+ builderModelFields,
4262
+ builderSampleEntries: builderEntries,
4263
+ existingFields,
4264
+ now: args.now,
4265
+ });
4266
+ }
3822
4267
 
3823
4268
  const itemsToLink = response.items.filter((item) =>
3824
4269
  builderEntriesByDocumentId.has(item.document.id),
3825
4270
  );
3826
4271
  const fetchedDocumentIds = itemsToLink.map((item) => item.document.id);
3827
4272
  if (fetchedDocumentIds.length > 0) {
3828
- await db
3829
- .delete(schema.contentDatabaseSourceRows)
3830
- .where(
3831
- and(
3832
- eq(schema.contentDatabaseSourceRows.sourceId, args.source.id),
3833
- inArray(
3834
- schema.contentDatabaseSourceRows.documentId,
3835
- fetchedDocumentIds,
4273
+ for (const idChunk of chunks(fetchedDocumentIds, idChunkSize())) {
4274
+ await db
4275
+ .delete(schema.contentDatabaseSourceRows)
4276
+ .where(
4277
+ and(
4278
+ eq(schema.contentDatabaseSourceRows.sourceId, args.source.id),
4279
+ inArray(schema.contentDatabaseSourceRows.documentId, idChunk),
3836
4280
  ),
3837
- ),
3838
- );
4281
+ );
4282
+ }
3839
4283
  await seedMockSourceRows({
3840
4284
  sourceId: args.source.id,
3841
4285
  ownerEmail: args.database.ownerEmail,
@@ -3868,17 +4312,24 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
3868
4312
  });
3869
4313
  }
3870
4314
  if (!hasMore && nextActiveReadSourceRowIds.length > 0) {
3871
- await db
3872
- .delete(schema.contentDatabaseSourceRows)
3873
- .where(
3874
- and(
3875
- eq(schema.contentDatabaseSourceRows.sourceId, args.source.id),
3876
- notInArray(
3877
- schema.contentDatabaseSourceRows.sourceRowId,
3878
- nextActiveReadSourceRowIds,
3879
- ),
3880
- ),
3881
- );
4315
+ const activeSourceRowIds = new Set(nextActiveReadSourceRowIds);
4316
+ const staleRows = (
4317
+ await db
4318
+ .select({
4319
+ id: schema.contentDatabaseSourceRows.id,
4320
+ sourceRowId: schema.contentDatabaseSourceRows.sourceRowId,
4321
+ })
4322
+ .from(schema.contentDatabaseSourceRows)
4323
+ .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id))
4324
+ ).filter((row) => !activeSourceRowIds.has(row.sourceRowId));
4325
+ for (const idChunk of chunks(
4326
+ staleRows.map((row) => row.id),
4327
+ idChunkSize(),
4328
+ )) {
4329
+ await db
4330
+ .delete(schema.contentDatabaseSourceRows)
4331
+ .where(inArray(schema.contentDatabaseSourceRows.id, idChunk));
4332
+ }
3882
4333
  }
3883
4334
 
3884
4335
  await updateBuilderCmsSourceReadMetadata({
@@ -3902,23 +4353,29 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
3902
4353
  return;
3903
4354
  }
3904
4355
 
3905
- await db
3906
- .delete(schema.contentDatabaseSourceFields)
3907
- .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
4356
+ const shouldReseedSourceFields =
4357
+ !builderModelFieldsReadFailed || existingFields.length === 0;
4358
+ if (shouldReseedSourceFields) {
4359
+ await db
4360
+ .delete(schema.contentDatabaseSourceFields)
4361
+ .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
4362
+ }
3908
4363
  await db
3909
4364
  .delete(schema.contentDatabaseSourceRows)
3910
4365
  .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
3911
4366
 
3912
- await seedMockSourceFields({
3913
- sourceId: args.source.id,
3914
- ownerEmail: args.database.ownerEmail,
3915
- sourceType: "builder-cms",
3916
- properties,
3917
- builderModelFields,
3918
- builderSampleEntries: builderEntries,
3919
- existingFields,
3920
- now: args.now,
3921
- });
4367
+ if (shouldReseedSourceFields) {
4368
+ await seedMockSourceFields({
4369
+ sourceId: args.source.id,
4370
+ ownerEmail: args.database.ownerEmail,
4371
+ sourceType: "builder-cms",
4372
+ properties,
4373
+ builderModelFields,
4374
+ builderSampleEntries: builderEntries,
4375
+ existingFields,
4376
+ now: args.now,
4377
+ });
4378
+ }
3922
4379
  // Row-union: a resync must only (re)link items that BELONG to this source —
3923
4380
  // never claim every database item. With a single source, all items belong to
3924
4381
  // it (back-compat). With multiple sources, link only this source's