@agent-native/core 0.133.1 → 0.133.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +12 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/a2a/client.ts +22 -5
- package/corpus/core/src/client/use-action.ts +6 -0
- package/corpus/core/src/integrations/webhook-handler.ts +7 -0
- package/corpus/core/src/scripts/call-agent.ts +29 -2
- package/corpus/templates/content/actions/_batch-utils.ts +33 -0
- package/corpus/templates/content/actions/_builder-cms-read-client.ts +173 -81
- package/corpus/templates/content/actions/_content-files.ts +31 -21
- package/corpus/templates/content/actions/_database-source-utils.ts +653 -155
- package/corpus/templates/content/actions/_database-utils.ts +535 -41
- package/corpus/templates/content/actions/add-content-database-source-field-property.ts +291 -240
- package/corpus/templates/content/actions/add-database-item.ts +15 -1
- package/corpus/templates/content/actions/attach-content-database-source.ts +156 -58
- package/corpus/templates/content/actions/bind-content-database-source-field.ts +2 -2
- package/corpus/templates/content/actions/cancel-prepared-builder-source-update.ts +4 -1
- package/corpus/templates/content/actions/delete-database-items.ts +8 -2
- package/corpus/templates/content/actions/disconnect-content-database-source.ts +2 -2
- package/corpus/templates/content/actions/duplicate-database-item.ts +15 -1
- package/corpus/templates/content/actions/duplicate-database-items.ts +19 -1
- package/corpus/templates/content/actions/execute-builder-source-execution.ts +2 -1
- package/corpus/templates/content/actions/get-content-database.ts +12 -64
- package/corpus/templates/content/actions/move-database-item.ts +4 -1
- package/corpus/templates/content/actions/prepare-builder-source-execution.ts +1 -1
- package/corpus/templates/content/actions/prepare-builder-source-review.ts +4 -8
- package/corpus/templates/content/actions/preview-content-database-source-attach.ts +92 -0
- package/corpus/templates/content/actions/process-builder-body-hydration.ts +2 -1
- package/corpus/templates/content/actions/query-content-database-items.ts +64 -0
- package/corpus/templates/content/actions/refresh-content-database-source.ts +7 -0
- package/corpus/templates/content/actions/review-content-database-source-change-set.ts +1 -1
- package/corpus/templates/content/actions/set-content-database-source-write-mode.ts +1 -1
- package/corpus/templates/content/actions/stage-builder-revision.ts +1 -1
- package/corpus/templates/content/actions/update-content-database-personal-view.ts +29 -3
- package/corpus/templates/content/actions/update-content-database-view.ts +1 -1
- package/corpus/templates/content/actions/validate-builder-source-execution.ts +1 -1
- package/corpus/templates/content/app/components/editor/DocumentProperties.tsx +3 -22
- package/corpus/templates/content/app/components/editor/database/DatabaseView.tsx +267 -105
- package/corpus/templates/content/app/components/editor/database/sidebar.tsx +1 -34
- package/corpus/templates/content/app/components/sidebar/DocumentSidebar.tsx +159 -4
- package/corpus/templates/content/app/components/sidebar/DocumentTreeItem.tsx +4 -211
- package/corpus/templates/content/app/hooks/use-content-database.ts +285 -24
- package/corpus/templates/content/app/hooks/use-document-properties.ts +7 -6
- package/corpus/templates/content/app/i18n-data.ts +10 -0
- package/corpus/templates/content/changelog/2026-07-29-large-databases-keep-useful-rows-visible.md +6 -0
- package/corpus/templates/content/changelog/2026-07-30-builder-source-columns-now-appear-immediately-when-connected.md +6 -0
- package/corpus/templates/content/changelog/2026-07-30-deleting-a-page-no-longer-leaves-the-content-sidebar-unrespo.md +6 -0
- package/corpus/templates/content/changelog/2026-07-30-large-builder-backed-tables-show-useful-rows-sooner-and-fini.md +6 -0
- package/corpus/templates/content/changelog/2026-08-01-large-builder-databases-now-show-rows-immediately-and-finish.md +6 -0
- package/corpus/templates/content/e2e/playwright.config.ts +2 -1
- package/corpus/templates/content/parity/matrix.md +2 -1
- package/corpus/templates/content/parity/matrix.ts +25 -0
- package/corpus/templates/content/shared/api.ts +54 -4
- package/corpus/templates/content/shared/database-query.ts +359 -0
- package/dist/a2a/client.d.ts +9 -1
- package/dist/a2a/client.d.ts.map +1 -1
- package/dist/a2a/client.js +12 -4
- package/dist/a2a/client.js.map +1 -1
- package/dist/client/use-action.d.ts.map +1 -1
- package/dist/client/use-action.js +6 -0
- package/dist/client/use-action.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/integrations/webhook-handler.d.ts.map +1 -1
- package/dist/integrations/webhook-handler.js +3 -0
- package/dist/integrations/webhook-handler.js.map +1 -1
- package/dist/observability/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/scripts/call-agent.d.ts.map +1 -1
- package/dist/scripts/call-agent.js +24 -2
- package/dist/scripts/call-agent.js.map +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/package.json +1 -1
- package/src/a2a/client.ts +22 -5
- package/src/client/use-action.ts +6 -0
- package/src/integrations/webhook-handler.ts +7 -0
- package/src/scripts/call-agent.ts +29 -2
package/corpus/README.md
CHANGED
package/corpus/core/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @agent-native/core
|
|
2
2
|
|
|
3
|
+
## 0.133.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 9258da4: Preserve nested object parameters when browser clients call GET actions.
|
|
8
|
+
|
|
9
|
+
## 0.133.2
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- 3fac05d: Ensure Netlify-hosted integration calls hand off slow cross-app work to durable delivery when only runtime markers are available.
|
|
14
|
+
|
|
3
15
|
## 0.133.1
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/corpus/core/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.133.
|
|
3
|
+
"version": "0.133.3",
|
|
4
4
|
"description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -536,8 +536,14 @@ export class A2AClient {
|
|
|
536
536
|
metadata?: Record<string, unknown>;
|
|
537
537
|
idempotencyKey?: string;
|
|
538
538
|
approvedActions?: A2AApprovedAction[];
|
|
539
|
-
/**
|
|
539
|
+
/** Time to wait after submission for completion. Default 5 min. */
|
|
540
540
|
timeoutMs?: number;
|
|
541
|
+
/**
|
|
542
|
+
* Optional separate budget for agent-card discovery and the initial
|
|
543
|
+
* async message submission. When omitted, timeoutMs remains the shared
|
|
544
|
+
* end-to-end deadline for backwards compatibility.
|
|
545
|
+
*/
|
|
546
|
+
submissionTimeoutMs?: number;
|
|
541
547
|
/** Poll interval. Default 2s. */
|
|
542
548
|
pollIntervalMs?: number;
|
|
543
549
|
/** Called with each polled task — useful for surfacing progress. */
|
|
@@ -545,7 +551,8 @@ export class A2AClient {
|
|
|
545
551
|
},
|
|
546
552
|
): Promise<Task> {
|
|
547
553
|
const timeoutMs = opts?.timeoutMs ?? 5 * 60_000;
|
|
548
|
-
const
|
|
554
|
+
const submissionDeadlineMs =
|
|
555
|
+
Date.now() + (opts?.submissionTimeoutMs ?? timeoutMs);
|
|
549
556
|
const submitted = await this.send(message, {
|
|
550
557
|
contextId: opts?.contextId,
|
|
551
558
|
metadata: opts?.metadata,
|
|
@@ -556,12 +563,19 @@ export class A2AClient {
|
|
|
556
563
|
async: true,
|
|
557
564
|
requestTimeoutMs: Math.min(
|
|
558
565
|
this.requestTimeoutMs ?? DEFAULT_A2A_POLL_REQUEST_TIMEOUT_MS,
|
|
559
|
-
Math.max(1,
|
|
566
|
+
Math.max(1, submissionDeadlineMs - Date.now()),
|
|
560
567
|
),
|
|
561
|
-
deadlineMs,
|
|
568
|
+
deadlineMs: submissionDeadlineMs,
|
|
562
569
|
});
|
|
563
570
|
|
|
564
|
-
|
|
571
|
+
const pollingDeadlineMs = opts?.submissionTimeoutMs
|
|
572
|
+
? Date.now() + timeoutMs
|
|
573
|
+
: submissionDeadlineMs;
|
|
574
|
+
return this.pollTask(submitted, {
|
|
575
|
+
...opts,
|
|
576
|
+
timeoutMs,
|
|
577
|
+
deadlineMs: pollingDeadlineMs,
|
|
578
|
+
});
|
|
565
579
|
}
|
|
566
580
|
|
|
567
581
|
/**
|
|
@@ -1017,6 +1031,8 @@ export async function callAgent(
|
|
|
1017
1031
|
async?: boolean;
|
|
1018
1032
|
/** Total time to wait for the polled task (default 5 min). */
|
|
1019
1033
|
timeoutMs?: number;
|
|
1034
|
+
/** Separate budget for discovery and initial async submission. */
|
|
1035
|
+
submissionTimeoutMs?: number;
|
|
1020
1036
|
/**
|
|
1021
1037
|
* Existing async task to keep polling. When set, no new message is sent.
|
|
1022
1038
|
* This prevents a caller-side timeout from duplicating downstream work.
|
|
@@ -1092,6 +1108,7 @@ export async function callAgent(
|
|
|
1092
1108
|
? { approvedActions: opts.approvedActions }
|
|
1093
1109
|
: {}),
|
|
1094
1110
|
timeoutMs: opts?.timeoutMs,
|
|
1111
|
+
submissionTimeoutMs: opts?.submissionTimeoutMs,
|
|
1095
1112
|
pollIntervalMs: opts?.pollIntervalMs,
|
|
1096
1113
|
onUpdate: opts?.onUpdate,
|
|
1097
1114
|
});
|
|
@@ -207,6 +207,12 @@ function appendActionQueryParam(
|
|
|
207
207
|
}
|
|
208
208
|
return;
|
|
209
209
|
}
|
|
210
|
+
if (typeof value === "object") {
|
|
211
|
+
// defineAction restores JSON strings when the schema expects an object.
|
|
212
|
+
// Preserve nested GET params instead of collapsing them to "[object Object]".
|
|
213
|
+
qs.append(key, JSON.stringify(value));
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
210
216
|
qs.append(key, String(value));
|
|
211
217
|
}
|
|
212
218
|
|
|
@@ -2274,6 +2274,13 @@ function isQueuedA2AContinuationDeferral(text: string): boolean {
|
|
|
2274
2274
|
if (!normalized) return true;
|
|
2275
2275
|
if (hasSubstantiveA2APartialAnswer(text)) return false;
|
|
2276
2276
|
if (normalized.includes(A2A_CONTINUATION_QUEUED_MARKER)) return true;
|
|
2277
|
+
if (
|
|
2278
|
+
/\bwill\b[^.!?]{0,160}\bpost\b[^.!?]{0,160}\b(?:thread|result|link|content id)\b/i.test(
|
|
2279
|
+
normalized,
|
|
2280
|
+
)
|
|
2281
|
+
) {
|
|
2282
|
+
return true;
|
|
2283
|
+
}
|
|
2277
2284
|
return /\b(?:still (?:working|processing)|is working on|taking longer than expected|will (?:post|update|surface|show up)|(?:it'?ll|it will|the result will|the final result will) (?:post|be posted|update|be updated|surface|show up)|will be (?:posted|updated|sent|shared)|final result when it finishes|while you wait|as soon as (?:it|it'?s|it is|the result|the artifact) (?:comes back|is ready|ready)|hang tight|relay from the .* agent)\b/i.test(
|
|
2278
2285
|
normalized,
|
|
2279
2286
|
);
|
|
@@ -38,6 +38,7 @@ import { track } from "../tracking/registry.js";
|
|
|
38
38
|
|
|
39
39
|
const DEFAULT_SERVERLESS_INTEGRATION_A2A_TIMEOUT_MS = 18_000;
|
|
40
40
|
const NETLIFY_INTEGRATION_A2A_TIMEOUT_MS = 2_000;
|
|
41
|
+
const NETLIFY_INTEGRATION_A2A_SUBMISSION_TIMEOUT_MS = 15_000;
|
|
41
42
|
const INTEGRATION_A2A_TOKEN_TTL = "30m";
|
|
42
43
|
const A2A_INVOCATION_EVENT = "$a2a_invocation";
|
|
43
44
|
|
|
@@ -167,13 +168,32 @@ function parseTimeoutMs(value: string | undefined): number | undefined {
|
|
|
167
168
|
return Math.floor(parsed);
|
|
168
169
|
}
|
|
169
170
|
|
|
171
|
+
function hasExplicitNonHostedNetlifyOverride(): boolean {
|
|
172
|
+
return (
|
|
173
|
+
process.env.NETLIFY_LOCAL === "true" || process.env.NETLIFY === "false"
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function isNetlifyHostedRuntimeForIntegrationCall(): boolean {
|
|
178
|
+
if (hasExplicitNonHostedNetlifyOverride()) return false;
|
|
179
|
+
if (process.env.NETLIFY && process.env.NETLIFY !== "false") return true;
|
|
180
|
+
|
|
181
|
+
// NETLIFY is a build-time marker, while deployed Netlify Functions expose
|
|
182
|
+
// SITE_ID at runtime. Recognize the same runtime-only marker used by the
|
|
183
|
+
// durable background and run-manager gates so the integration caller hands
|
|
184
|
+
// slow A2A work to durable delivery before its foreground budget expires.
|
|
185
|
+
return Boolean(process.env.SITE_ID); // guard:allow-env-credential -- Netlify's read-only public site identifier is a runtime host marker, not a user credential.
|
|
186
|
+
}
|
|
187
|
+
|
|
170
188
|
function isServerlessHost(): boolean {
|
|
189
|
+
if (hasExplicitNonHostedNetlifyOverride()) return false;
|
|
190
|
+
|
|
171
191
|
// Detection mirrors db/migrations.ts:297-301. On Cloudflare Workers/Pages,
|
|
172
192
|
// `process.env` is shimmed and CF_PAGES isn't reliably populated at runtime —
|
|
173
193
|
// the canonical signal is the `__cf_env`/`__env__` global injected by the
|
|
174
194
|
// Cloudflare runtime adapter.
|
|
175
195
|
return (
|
|
176
|
-
|
|
196
|
+
isNetlifyHostedRuntimeForIntegrationCall() ||
|
|
177
197
|
!!process.env.AWS_LAMBDA_FUNCTION_NAME ||
|
|
178
198
|
!!process.env.VERCEL ||
|
|
179
199
|
"__cf_env" in globalThis ||
|
|
@@ -193,7 +213,9 @@ function getIntegrationCallTimeoutMs(): number | undefined {
|
|
|
193
213
|
// calls very short so multi-agent integration requests queue downstream
|
|
194
214
|
// continuations quickly instead of spending the parent Slack/email processor
|
|
195
215
|
// budget waiting on separately deployed apps one-by-one.
|
|
196
|
-
if (
|
|
216
|
+
if (isNetlifyHostedRuntimeForIntegrationCall()) {
|
|
217
|
+
return NETLIFY_INTEGRATION_A2A_TIMEOUT_MS;
|
|
218
|
+
}
|
|
197
219
|
|
|
198
220
|
return DEFAULT_SERVERLESS_INTEGRATION_A2A_TIMEOUT_MS;
|
|
199
221
|
}
|
|
@@ -676,6 +698,10 @@ export async function run(
|
|
|
676
698
|
// Docker can wait for slow-but-valid answers; integration processors
|
|
677
699
|
// still need to finish before their current function execution dies.
|
|
678
700
|
const callTimeoutMs = getIntegrationCallTimeoutMs();
|
|
701
|
+
const submissionTimeoutMs =
|
|
702
|
+
callTimeoutMs && isNetlifyHostedRuntimeForIntegrationCall()
|
|
703
|
+
? NETLIFY_INTEGRATION_A2A_SUBMISSION_TIMEOUT_MS
|
|
704
|
+
: undefined;
|
|
679
705
|
responseText = await callAgent(agent.url, messageWithHint, {
|
|
680
706
|
apiKey,
|
|
681
707
|
userEmail: callerEmail,
|
|
@@ -692,6 +718,7 @@ export async function run(
|
|
|
692
718
|
...(callTimeoutMs
|
|
693
719
|
? {
|
|
694
720
|
timeoutMs: callTimeoutMs,
|
|
721
|
+
...(submissionTimeoutMs ? { submissionTimeoutMs } : {}),
|
|
695
722
|
}
|
|
696
723
|
: {}),
|
|
697
724
|
});
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { getDialect, type Dialect } from "@agent-native/core/db";
|
|
2
|
+
|
|
1
3
|
export function chunks<T>(items: T[], size: number): T[][] {
|
|
2
4
|
if (items.length === 0) return [];
|
|
3
5
|
const out: T[][] = [];
|
|
@@ -6,3 +8,34 @@ export function chunks<T>(items: T[], size: number): T[][] {
|
|
|
6
8
|
}
|
|
7
9
|
return out;
|
|
8
10
|
}
|
|
11
|
+
|
|
12
|
+
export function bulkChunkSizeForColumnCount(
|
|
13
|
+
columnCount: number,
|
|
14
|
+
dialect: Dialect = getDialect(),
|
|
15
|
+
) {
|
|
16
|
+
// D1 rejects statements with more than 100 bound params, so derive every
|
|
17
|
+
// bulk chunk from the statement's column count instead of a fixed row count.
|
|
18
|
+
const budget = dialect === "d1" ? 90 : 900;
|
|
19
|
+
return Math.max(1, Math.floor(budget / Math.max(1, columnCount)));
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function processWithConcurrency<T>(
|
|
23
|
+
items: readonly T[],
|
|
24
|
+
concurrency: number,
|
|
25
|
+
worker: (item: T) => Promise<void>,
|
|
26
|
+
) {
|
|
27
|
+
let nextIndex = 0;
|
|
28
|
+
const workerCount = Math.min(
|
|
29
|
+
items.length,
|
|
30
|
+
Math.max(1, Math.floor(concurrency)),
|
|
31
|
+
);
|
|
32
|
+
await Promise.all(
|
|
33
|
+
Array.from({ length: workerCount }, async () => {
|
|
34
|
+
while (nextIndex < items.length) {
|
|
35
|
+
const item = items[nextIndex];
|
|
36
|
+
nextIndex += 1;
|
|
37
|
+
if (item !== undefined) await worker(item);
|
|
38
|
+
}
|
|
39
|
+
}),
|
|
40
|
+
);
|
|
41
|
+
}
|
|
@@ -163,6 +163,7 @@ type BuilderMcpToolResult = {
|
|
|
163
163
|
const BUILDER_CMS_DEFAULT_READ_LIMIT = 500;
|
|
164
164
|
const BUILDER_CMS_MAX_READ_LIMIT = 10_000;
|
|
165
165
|
const BUILDER_CMS_PAGE_SIZE = 100;
|
|
166
|
+
const BUILDER_CMS_PROJECTED_READ_CONCURRENCY = 8;
|
|
166
167
|
const BUILDER_CMS_READ_RETRIES = 2;
|
|
167
168
|
const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [
|
|
168
169
|
"id",
|
|
@@ -192,6 +193,10 @@ const BUILDER_CMS_HEAVY_BODY_FIELD_PATHS = [
|
|
|
192
193
|
const BUILDER_CMS_FIELD_PATH_PATTERN =
|
|
193
194
|
/^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/;
|
|
194
195
|
|
|
196
|
+
type BuilderCmsContentPageResult =
|
|
197
|
+
| { pageLimit: number; pageEntries: BuilderCmsSourceEntry[] }
|
|
198
|
+
| { pageLimit: number; error: string };
|
|
199
|
+
|
|
195
200
|
function normalizeBuilderCmsListFieldPath(fieldPath: string) {
|
|
196
201
|
const trimmed = fieldPath.trim();
|
|
197
202
|
if (!trimmed || !BUILDER_CMS_FIELD_PATH_PATTERN.test(trimmed)) return null;
|
|
@@ -218,6 +223,36 @@ function normalizeBuilderCmsListFieldPath(fieldPath: string) {
|
|
|
218
223
|
return normalized;
|
|
219
224
|
}
|
|
220
225
|
|
|
226
|
+
function preserveProjectedBuilderFieldAbsence(
|
|
227
|
+
read: BuilderCmsReadResult,
|
|
228
|
+
fieldPaths: readonly string[] | undefined,
|
|
229
|
+
rawData: boolean | undefined,
|
|
230
|
+
): BuilderCmsReadResult {
|
|
231
|
+
if (read.state !== "live" || rawData === true || !fieldPaths?.length) {
|
|
232
|
+
return read;
|
|
233
|
+
}
|
|
234
|
+
const projectedFieldPaths = [
|
|
235
|
+
...new Set(
|
|
236
|
+
fieldPaths
|
|
237
|
+
.map(normalizeBuilderCmsListFieldPath)
|
|
238
|
+
.filter((fieldPath): fieldPath is string => fieldPath !== null),
|
|
239
|
+
),
|
|
240
|
+
];
|
|
241
|
+
if (projectedFieldPaths.length === 0) return read;
|
|
242
|
+
return {
|
|
243
|
+
...read,
|
|
244
|
+
entries: read.entries.map((entry) => {
|
|
245
|
+
const sourceValues = { ...entry.sourceValues };
|
|
246
|
+
for (const fieldPath of projectedFieldPaths) {
|
|
247
|
+
if (!Object.prototype.hasOwnProperty.call(sourceValues, fieldPath)) {
|
|
248
|
+
sourceValues[fieldPath] = null;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return { ...entry, sourceValues };
|
|
252
|
+
}),
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
|
|
221
256
|
export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) {
|
|
222
257
|
const fields = new Map<string, string>();
|
|
223
258
|
for (const fieldPath of [
|
|
@@ -752,6 +787,7 @@ async function initializeBuilderMcp(args: {
|
|
|
752
787
|
async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
753
788
|
model: string;
|
|
754
789
|
fieldPaths?: readonly string[];
|
|
790
|
+
includeBodies?: boolean;
|
|
755
791
|
rawData?: boolean;
|
|
756
792
|
limit?: number;
|
|
757
793
|
maxPages?: number;
|
|
@@ -775,7 +811,9 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
|
775
811
|
const contentEntries: BuilderCmsSourceEntry[] = [];
|
|
776
812
|
const fields = args.rawData
|
|
777
813
|
? undefined
|
|
778
|
-
:
|
|
814
|
+
: args.includeBodies
|
|
815
|
+
? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}`
|
|
816
|
+
: builderCmsListEntryFields(args.fieldPaths);
|
|
779
817
|
const seenContentIds = new Set<string>();
|
|
780
818
|
let pagesRead = 0;
|
|
781
819
|
let hasMore = false;
|
|
@@ -941,6 +979,8 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
|
941
979
|
async function readBuilderCmsContentEntriesViaContentApi(args: {
|
|
942
980
|
model: string;
|
|
943
981
|
fieldPaths?: readonly string[];
|
|
982
|
+
includeBodies?: boolean;
|
|
983
|
+
allowCached?: boolean;
|
|
944
984
|
rawData?: boolean;
|
|
945
985
|
limit?: number;
|
|
946
986
|
maxPages?: number;
|
|
@@ -959,14 +999,21 @@ async function readBuilderCmsContentEntriesViaContentApi(args: {
|
|
|
959
999
|
// Fidelity reads intentionally do neither: Builder's raw data object is the
|
|
960
1000
|
// clone contract, including unresolved references and every nested field.
|
|
961
1001
|
url.searchParams.set("enrich", args.rawData === true ? "false" : "true");
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
1002
|
+
if (args.allowCached !== true || args.rawData === true) {
|
|
1003
|
+
url.searchParams.set("noCache", "true");
|
|
1004
|
+
url.searchParams.set(
|
|
1005
|
+
"cachebust",
|
|
1006
|
+
args.rawData === true ? String(Date.now()) : "true",
|
|
1007
|
+
);
|
|
1008
|
+
}
|
|
967
1009
|
url.searchParams.set("includeUnpublished", "true");
|
|
968
1010
|
if (args.rawData !== true) {
|
|
969
|
-
url.searchParams.set(
|
|
1011
|
+
url.searchParams.set(
|
|
1012
|
+
"fields",
|
|
1013
|
+
args.includeBodies
|
|
1014
|
+
? `${builderCmsListEntryFields(args.fieldPaths)},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}`
|
|
1015
|
+
: builderCmsListEntryFields(args.fieldPaths),
|
|
1016
|
+
);
|
|
970
1017
|
}
|
|
971
1018
|
|
|
972
1019
|
const limit = readLimit(args.limit);
|
|
@@ -978,73 +1025,103 @@ async function readBuilderCmsContentEntriesViaContentApi(args: {
|
|
|
978
1025
|
const seenIds = new Set<string>();
|
|
979
1026
|
let pagesRead = 0;
|
|
980
1027
|
let hasMore = false;
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1028
|
+
const concurrency =
|
|
1029
|
+
args.rawData === true ? 1 : BUILDER_CMS_PROJECTED_READ_CONCURRENCY;
|
|
1030
|
+
const errorResult = (message: string): BuilderCmsReadResult => ({
|
|
1031
|
+
state: "error",
|
|
1032
|
+
entries: [],
|
|
1033
|
+
fetchedAt,
|
|
1034
|
+
message,
|
|
1035
|
+
progress: {
|
|
1036
|
+
requestedLimit: limit,
|
|
1037
|
+
pageSize: BUILDER_CMS_PAGE_SIZE,
|
|
1038
|
+
startOffset,
|
|
1039
|
+
nextOffset: startOffset + entries.length,
|
|
1040
|
+
fetchedEntryCount: startOffset + entries.length,
|
|
1041
|
+
hasMore,
|
|
1042
|
+
partial: Boolean(args.maxPages) && hasMore,
|
|
1043
|
+
readMode: "builder-api",
|
|
1044
|
+
},
|
|
1045
|
+
});
|
|
990
1046
|
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1047
|
+
while (entries.length < limit) {
|
|
1048
|
+
const pagesRemaining = args.maxPages
|
|
1049
|
+
? args.maxPages - pagesRead
|
|
1050
|
+
: Number.POSITIVE_INFINITY;
|
|
1051
|
+
if (pagesRemaining <= 0) break;
|
|
1052
|
+
const pageCount = Math.min(
|
|
1053
|
+
pagesRead === 0 ? 1 : concurrency,
|
|
1054
|
+
pagesRemaining,
|
|
1055
|
+
Math.ceil((limit - entries.length) / BUILDER_CMS_PAGE_SIZE),
|
|
1056
|
+
);
|
|
1057
|
+
const pageRequests = Array.from({ length: pageCount }, (_, index) => {
|
|
1058
|
+
const pageUrl = new URL(url);
|
|
1059
|
+
const pageLimit = readPageLimit(
|
|
1060
|
+
limit - entries.length - index * BUILDER_CMS_PAGE_SIZE,
|
|
1061
|
+
);
|
|
1062
|
+
pageUrl.searchParams.set("limit", String(pageLimit));
|
|
1063
|
+
pageUrl.searchParams.set(
|
|
1064
|
+
"offset",
|
|
1065
|
+
String(startOffset + (pagesRead + index) * BUILDER_CMS_PAGE_SIZE),
|
|
1066
|
+
);
|
|
1067
|
+
return { pageLimit, pageUrl };
|
|
1068
|
+
});
|
|
1069
|
+
const pageResults = await Promise.all(
|
|
1070
|
+
pageRequests.map(
|
|
1071
|
+
async ({
|
|
1072
|
+
pageLimit,
|
|
1073
|
+
pageUrl,
|
|
1074
|
+
}): Promise<BuilderCmsContentPageResult> => {
|
|
1075
|
+
try {
|
|
1076
|
+
const response = await fetchBuilderContentPage({
|
|
1077
|
+
fetchImpl: args.fetchImpl,
|
|
1078
|
+
url: pageUrl,
|
|
1079
|
+
privateKey: args.privateKey,
|
|
1080
|
+
});
|
|
1081
|
+
if (!response.ok) {
|
|
1082
|
+
return {
|
|
1083
|
+
pageLimit,
|
|
1084
|
+
error: `Builder CMS read failed with HTTP ${response.status}.`,
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
const json = (await response.json()) as unknown;
|
|
1088
|
+
const pageEntries = entryArrayFromResponse(json)
|
|
1089
|
+
.map((entry) => normalizeBuilderCmsApiEntry(entry, args.model))
|
|
1090
|
+
.filter((entry): entry is BuilderCmsSourceEntry =>
|
|
1091
|
+
Boolean(entry),
|
|
1092
|
+
);
|
|
1093
|
+
return { pageLimit, pageEntries };
|
|
1094
|
+
} catch (error) {
|
|
1095
|
+
return {
|
|
1096
|
+
pageLimit,
|
|
1097
|
+
error:
|
|
1098
|
+
error instanceof Error
|
|
1099
|
+
? `Builder CMS read failed: ${error.message}`
|
|
1100
|
+
: "Builder CMS read failed.",
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1016
1103
|
},
|
|
1017
|
-
|
|
1018
|
-
|
|
1104
|
+
),
|
|
1105
|
+
);
|
|
1019
1106
|
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
};
|
|
1107
|
+
let stoppedOnShortPage = false;
|
|
1108
|
+
for (const pageResult of pageResults) {
|
|
1109
|
+
if ("error" in pageResult) return errorResult(pageResult.error);
|
|
1110
|
+
const appended = appendUniqueBuilderEntries(
|
|
1111
|
+
entries,
|
|
1112
|
+
seenIds,
|
|
1113
|
+
pageResult.pageEntries,
|
|
1114
|
+
);
|
|
1115
|
+
pagesRead += 1;
|
|
1116
|
+
hasMore =
|
|
1117
|
+
pageResult.pageEntries.length >= pageResult.pageLimit && appended > 0;
|
|
1118
|
+
if (args.maxPages && pagesRead >= args.maxPages) break;
|
|
1119
|
+
if (!hasMore) {
|
|
1120
|
+
stoppedOnShortPage = true;
|
|
1121
|
+
break;
|
|
1122
|
+
}
|
|
1037
1123
|
}
|
|
1038
|
-
|
|
1039
|
-
const json = (await response.json()) as unknown;
|
|
1040
|
-
const pageEntries = entryArrayFromResponse(json)
|
|
1041
|
-
.map((entry) => normalizeBuilderCmsApiEntry(entry, args.model))
|
|
1042
|
-
.filter((entry): entry is BuilderCmsSourceEntry => Boolean(entry));
|
|
1043
|
-
const appended = appendUniqueBuilderEntries(entries, seenIds, pageEntries);
|
|
1044
|
-
pagesRead += 1;
|
|
1045
|
-
hasMore = pageEntries.length >= pageLimit && appended > 0;
|
|
1046
|
-
if (args.maxPages && pagesRead >= args.maxPages) break;
|
|
1047
|
-
if (!hasMore) break;
|
|
1124
|
+
if (stoppedOnShortPage || !hasMore) break;
|
|
1048
1125
|
}
|
|
1049
1126
|
|
|
1050
1127
|
return {
|
|
@@ -1231,6 +1308,8 @@ export async function readBuilderCmsModelFields(args: {
|
|
|
1231
1308
|
export async function readBuilderCmsContentEntries(args: {
|
|
1232
1309
|
model: string;
|
|
1233
1310
|
fieldPaths?: readonly string[];
|
|
1311
|
+
includeBodies?: boolean;
|
|
1312
|
+
allowCached?: boolean;
|
|
1234
1313
|
rawData?: boolean;
|
|
1235
1314
|
requirePrivateKey?: boolean;
|
|
1236
1315
|
limit?: number;
|
|
@@ -1265,6 +1344,8 @@ export async function readBuilderCmsContentEntries(args: {
|
|
|
1265
1344
|
const contentApiRead = await readBuilderCmsContentEntriesViaContentApi({
|
|
1266
1345
|
model: args.model,
|
|
1267
1346
|
fieldPaths: args.fieldPaths,
|
|
1347
|
+
includeBodies: args.includeBodies,
|
|
1348
|
+
allowCached: args.allowCached,
|
|
1268
1349
|
rawData: args.rawData,
|
|
1269
1350
|
limit: args.limit,
|
|
1270
1351
|
maxPages: args.maxPages,
|
|
@@ -1273,22 +1354,33 @@ export async function readBuilderCmsContentEntries(args: {
|
|
|
1273
1354
|
publicKey,
|
|
1274
1355
|
privateKey: privateKey ?? undefined,
|
|
1275
1356
|
});
|
|
1276
|
-
if (contentApiRead.state === "live")
|
|
1357
|
+
if (contentApiRead.state === "live") {
|
|
1358
|
+
return preserveProjectedBuilderFieldAbsence(
|
|
1359
|
+
contentApiRead,
|
|
1360
|
+
args.fieldPaths,
|
|
1361
|
+
args.rawData,
|
|
1362
|
+
);
|
|
1363
|
+
}
|
|
1277
1364
|
if (!privateKey) return contentApiRead;
|
|
1278
1365
|
}
|
|
1279
1366
|
|
|
1280
1367
|
if (privateKey) {
|
|
1281
1368
|
try {
|
|
1282
|
-
return
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1369
|
+
return preserveProjectedBuilderFieldAbsence(
|
|
1370
|
+
await readBuilderCmsContentEntriesViaMcp({
|
|
1371
|
+
model: args.model,
|
|
1372
|
+
fieldPaths: args.fieldPaths,
|
|
1373
|
+
includeBodies: args.includeBodies,
|
|
1374
|
+
rawData: args.rawData,
|
|
1375
|
+
limit: args.limit,
|
|
1376
|
+
maxPages: args.maxPages,
|
|
1377
|
+
offset: args.offset,
|
|
1378
|
+
fetchImpl,
|
|
1379
|
+
privateKey,
|
|
1380
|
+
}),
|
|
1381
|
+
args.fieldPaths,
|
|
1382
|
+
args.rawData,
|
|
1383
|
+
);
|
|
1292
1384
|
} catch (error) {
|
|
1293
1385
|
return {
|
|
1294
1386
|
state: "error",
|
|
@@ -4,6 +4,7 @@ import { accessFilter } from "@agent-native/core/sharing";
|
|
|
4
4
|
import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
|
|
5
5
|
|
|
6
6
|
import { schema } from "../server/db/index.js";
|
|
7
|
+
import { bulkChunkSizeForColumnCount, chunks } from "./_batch-utils.js";
|
|
7
8
|
import {
|
|
8
9
|
listContentOrganizationMemberships,
|
|
9
10
|
normalizeContentSpaceEmail,
|
|
@@ -32,23 +33,29 @@ export function contentFilesItemId(databaseId: string, documentId: string) {
|
|
|
32
33
|
|
|
33
34
|
async function remapItemReferences(db: Db, replacements: Map<string, string>) {
|
|
34
35
|
if (!replacements.size) return;
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
.
|
|
50
|
-
.
|
|
51
|
-
.
|
|
36
|
+
for (const replacementBatch of chunks(
|
|
37
|
+
[...replacements.entries()],
|
|
38
|
+
bulkChunkSizeForColumnCount(3),
|
|
39
|
+
)) {
|
|
40
|
+
const batchReplacements = new Map(replacementBatch);
|
|
41
|
+
const duplicateIds = [...batchReplacements.keys()];
|
|
42
|
+
const remap = (column: any) => {
|
|
43
|
+
let expression = sql`CASE ${column}`;
|
|
44
|
+
for (const [duplicateId, canonicalId] of batchReplacements)
|
|
45
|
+
expression = sql`${expression} WHEN ${duplicateId} THEN ${canonicalId}`;
|
|
46
|
+
return sql`${expression} ELSE ${column} END`;
|
|
47
|
+
};
|
|
48
|
+
for (const table of [
|
|
49
|
+
schema.contentSpaceCatalogItems,
|
|
50
|
+
schema.contentDatabaseBodyHydrationQueue,
|
|
51
|
+
schema.contentDatabaseSourceRows,
|
|
52
|
+
schema.contentDatabaseSourceChangeSets,
|
|
53
|
+
]) {
|
|
54
|
+
await db
|
|
55
|
+
.update(table)
|
|
56
|
+
.set({ databaseItemId: remap(table.databaseItemId) })
|
|
57
|
+
.where(inArray(table.databaseItemId, duplicateIds));
|
|
58
|
+
}
|
|
52
59
|
}
|
|
53
60
|
}
|
|
54
61
|
|
|
@@ -170,15 +177,18 @@ async function reconcileDocuments(args: {
|
|
|
170
177
|
}
|
|
171
178
|
}
|
|
172
179
|
await remapItemReferences(args.db, replacements);
|
|
173
|
-
|
|
180
|
+
for (const deleteBatch of chunks(
|
|
181
|
+
[...deleteIds],
|
|
182
|
+
bulkChunkSizeForColumnCount(1),
|
|
183
|
+
)) {
|
|
174
184
|
await args.db
|
|
175
185
|
.delete(schema.contentDatabaseItems)
|
|
176
|
-
.where(inArray(schema.contentDatabaseItems.id,
|
|
186
|
+
.where(inArray(schema.contentDatabaseItems.id, deleteBatch));
|
|
177
187
|
}
|
|
178
|
-
|
|
188
|
+
for (const insertBatch of chunks(inserts, bulkChunkSizeForColumnCount(8))) {
|
|
179
189
|
await args.db
|
|
180
190
|
.insert(schema.contentDatabaseItems)
|
|
181
|
-
.values(
|
|
191
|
+
.values(insertBatch)
|
|
182
192
|
.onConflictDoNothing();
|
|
183
193
|
}
|
|
184
194
|
return { inserted: inserts.length, removed: deleteIds.size };
|