@agent-native/core 0.92.11 → 0.92.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +6 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/production-agent.ts +33 -2
- package/corpus/core/src/agent/run-manager.ts +41 -10
- package/corpus/core/src/agent/run-store.ts +72 -27
- package/corpus/core/src/client/agent-chat-adapter.ts +56 -12
- package/corpus/templates/clips/app/components/player/transcript-panel.tsx +27 -5
- package/corpus/templates/clips/app/hooks/use-player-shortcuts.ts +1 -0
- package/corpus/templates/clips/changelog/2026-07-10-fixed-copying-agent-responses-and-transcript-text-from-recor.md +6 -0
- package/corpus/templates/content/actions/_builder-cms-read-client.ts +82 -7
- package/corpus/templates/content/actions/_database-source-utils.ts +149 -30
- package/corpus/templates/content/actions/add-content-database-source-field-property.ts +4 -7
- package/corpus/templates/content/actions/attach-content-database-source.ts +41 -17
- package/corpus/templates/content/actions/change-content-database-source-role.ts +46 -8
- package/corpus/templates/content/actions/suggest-source-join-key.ts +8 -1
- package/corpus/templates/content/changelog/2026-07-09-builder-source-refreshes-now-keep-mapped-fields-such-as-topi.md +6 -0
- package/corpus/templates/content/shared/api.ts +1 -0
- package/dist/agent/production-agent.d.ts +3 -1
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +19 -3
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +34 -9
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +68 -22
- package/dist/agent/run-store.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +48 -12
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/observability/routes.d.ts +5 -5
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +3 -3
- package/package.json +1 -1
|
@@ -59,9 +59,75 @@ const BUILDER_CMS_DEFAULT_READ_LIMIT = 500;
|
|
|
59
59
|
const BUILDER_CMS_MAX_READ_LIMIT = 1000;
|
|
60
60
|
const BUILDER_CMS_PAGE_SIZE = 100;
|
|
61
61
|
const BUILDER_CMS_READ_RETRIES = 2;
|
|
62
|
-
const
|
|
63
|
-
"id,
|
|
64
|
-
|
|
62
|
+
const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [
|
|
63
|
+
"id",
|
|
64
|
+
"name",
|
|
65
|
+
"published",
|
|
66
|
+
"lastUpdated",
|
|
67
|
+
"createdDate",
|
|
68
|
+
"data.title",
|
|
69
|
+
"data.handle",
|
|
70
|
+
"data.url",
|
|
71
|
+
"data.slug",
|
|
72
|
+
"data.date",
|
|
73
|
+
"data.description",
|
|
74
|
+
"data.status",
|
|
75
|
+
"data.author",
|
|
76
|
+
"data.image",
|
|
77
|
+
] as const;
|
|
78
|
+
const BUILDER_CMS_TOP_LEVEL_METADATA_FIELDS = new Set(
|
|
79
|
+
BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS.filter(
|
|
80
|
+
(fieldPath) => !fieldPath.startsWith("data."),
|
|
81
|
+
).map((fieldPath) => fieldPath.toLowerCase()),
|
|
82
|
+
);
|
|
83
|
+
const BUILDER_CMS_HEAVY_BODY_FIELD_PATHS = [
|
|
84
|
+
"data.blocks",
|
|
85
|
+
"data.blocksString",
|
|
86
|
+
] as const;
|
|
87
|
+
const BUILDER_CMS_FIELD_PATH_PATTERN =
|
|
88
|
+
/^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/;
|
|
89
|
+
|
|
90
|
+
function normalizeBuilderCmsListFieldPath(fieldPath: string) {
|
|
91
|
+
const trimmed = fieldPath.trim();
|
|
92
|
+
if (!trimmed || !BUILDER_CMS_FIELD_PATH_PATTERN.test(trimmed)) return null;
|
|
93
|
+
const normalized = trimmed.includes(".")
|
|
94
|
+
? trimmed
|
|
95
|
+
: BUILDER_CMS_TOP_LEVEL_METADATA_FIELDS.has(trimmed.toLowerCase())
|
|
96
|
+
? trimmed
|
|
97
|
+
: `data.${trimmed}`;
|
|
98
|
+
const lower = normalized.toLowerCase();
|
|
99
|
+
if (
|
|
100
|
+
BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.some((heavyFieldPath) => {
|
|
101
|
+
const heavyLower = heavyFieldPath.toLowerCase();
|
|
102
|
+
return lower === heavyLower || lower.startsWith(`${heavyLower}.`);
|
|
103
|
+
})
|
|
104
|
+
) {
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
if (
|
|
108
|
+
normalized.includes(".") &&
|
|
109
|
+
!normalized.toLowerCase().startsWith("data.")
|
|
110
|
+
) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
return normalized;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) {
|
|
117
|
+
const fields = new Map<string, string>();
|
|
118
|
+
for (const fieldPath of [
|
|
119
|
+
...BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS,
|
|
120
|
+
...fieldPaths,
|
|
121
|
+
]) {
|
|
122
|
+
const normalized = normalizeBuilderCmsListFieldPath(fieldPath);
|
|
123
|
+
if (!normalized) continue;
|
|
124
|
+
if (!fields.has(normalized)) fields.set(normalized, normalized);
|
|
125
|
+
}
|
|
126
|
+
return Array.from(fields.values()).join(",");
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const BUILDER_CMS_METADATA_ENTRY_FIELDS = builderCmsListEntryFields();
|
|
130
|
+
const BUILDER_CMS_BODY_ENTRY_FIELDS = `${BUILDER_CMS_METADATA_ENTRY_FIELDS},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}`;
|
|
65
131
|
|
|
66
132
|
function builderContentApiHost() {
|
|
67
133
|
return (
|
|
@@ -444,6 +510,7 @@ async function initializeBuilderMcp(args: {
|
|
|
444
510
|
|
|
445
511
|
async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
446
512
|
model: string;
|
|
513
|
+
fieldPaths?: readonly string[];
|
|
447
514
|
limit?: number;
|
|
448
515
|
maxPages?: number;
|
|
449
516
|
offset?: number;
|
|
@@ -464,6 +531,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
|
464
531
|
? Math.max(0, Math.floor(args.offset))
|
|
465
532
|
: 0;
|
|
466
533
|
const contentEntries: BuilderCmsSourceEntry[] = [];
|
|
534
|
+
const fields = builderCmsListEntryFields(args.fieldPaths);
|
|
467
535
|
const seenContentIds = new Set<string>();
|
|
468
536
|
let pagesRead = 0;
|
|
469
537
|
let hasMore = false;
|
|
@@ -490,7 +558,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
|
490
558
|
modelName: args.model,
|
|
491
559
|
limit: pageLimit,
|
|
492
560
|
offset,
|
|
493
|
-
fields
|
|
561
|
+
fields,
|
|
494
562
|
enrich: true,
|
|
495
563
|
},
|
|
496
564
|
},
|
|
@@ -600,7 +668,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
|
600
668
|
modelName: args.model,
|
|
601
669
|
limit: 1,
|
|
602
670
|
query: { id: entry.id },
|
|
603
|
-
fields
|
|
671
|
+
fields,
|
|
604
672
|
enrich: true,
|
|
605
673
|
},
|
|
606
674
|
},
|
|
@@ -633,6 +701,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: {
|
|
|
633
701
|
|
|
634
702
|
async function readBuilderCmsContentEntriesViaContentApi(args: {
|
|
635
703
|
model: string;
|
|
704
|
+
fieldPaths?: readonly string[];
|
|
636
705
|
limit?: number;
|
|
637
706
|
maxPages?: number;
|
|
638
707
|
offset?: number;
|
|
@@ -650,7 +719,7 @@ async function readBuilderCmsContentEntriesViaContentApi(args: {
|
|
|
650
719
|
// bare reference id.
|
|
651
720
|
url.searchParams.set("enrich", "true");
|
|
652
721
|
url.searchParams.set("noCache", "true");
|
|
653
|
-
url.searchParams.set("fields",
|
|
722
|
+
url.searchParams.set("fields", builderCmsListEntryFields(args.fieldPaths));
|
|
654
723
|
|
|
655
724
|
const limit = readLimit(args.limit);
|
|
656
725
|
const startOffset =
|
|
@@ -902,7 +971,10 @@ export async function readBuilderCmsModelFields(args: {
|
|
|
902
971
|
fetchImpl?: FetchLike;
|
|
903
972
|
}): Promise<BuilderCmsModelFieldSummary[]> {
|
|
904
973
|
const models = await listBuilderCmsModels({ fetchImpl: args.fetchImpl });
|
|
905
|
-
if (models.state
|
|
974
|
+
if (models.state === "unconfigured") return [];
|
|
975
|
+
if (models.state === "error") {
|
|
976
|
+
throw new Error(models.message ?? "Builder CMS model discovery failed.");
|
|
977
|
+
}
|
|
906
978
|
const modelName = args.model.trim().toLowerCase();
|
|
907
979
|
return (
|
|
908
980
|
models.models.find((model) => {
|
|
@@ -917,6 +989,7 @@ export async function readBuilderCmsModelFields(args: {
|
|
|
917
989
|
|
|
918
990
|
export async function readBuilderCmsContentEntries(args: {
|
|
919
991
|
model: string;
|
|
992
|
+
fieldPaths?: readonly string[];
|
|
920
993
|
limit?: number;
|
|
921
994
|
maxPages?: number;
|
|
922
995
|
offset?: number;
|
|
@@ -929,6 +1002,7 @@ export async function readBuilderCmsContentEntries(args: {
|
|
|
929
1002
|
if (publicKey) {
|
|
930
1003
|
const contentApiRead = await readBuilderCmsContentEntriesViaContentApi({
|
|
931
1004
|
model: args.model,
|
|
1005
|
+
fieldPaths: args.fieldPaths,
|
|
932
1006
|
limit: args.limit,
|
|
933
1007
|
maxPages: args.maxPages,
|
|
934
1008
|
offset: args.offset,
|
|
@@ -945,6 +1019,7 @@ export async function readBuilderCmsContentEntries(args: {
|
|
|
945
1019
|
try {
|
|
946
1020
|
return await readBuilderCmsContentEntriesViaMcp({
|
|
947
1021
|
model: args.model,
|
|
1022
|
+
fieldPaths: args.fieldPaths,
|
|
948
1023
|
limit: args.limit,
|
|
949
1024
|
maxPages: args.maxPages,
|
|
950
1025
|
offset: args.offset,
|
|
@@ -128,6 +128,7 @@ type SourceMetadataRecord = {
|
|
|
128
128
|
lastReadPartial?: boolean;
|
|
129
129
|
lastReadHasMore?: boolean;
|
|
130
130
|
lastReadNextOffset?: number;
|
|
131
|
+
lastReadSuspiciousEmpty?: boolean;
|
|
131
132
|
activeReadSourceRowIds?: string[];
|
|
132
133
|
sourceFetchState?: "idle" | "fetching" | "error" | string;
|
|
133
134
|
allowDraftWrites?: boolean;
|
|
@@ -336,6 +337,48 @@ const HEAVY_BUILDER_BODY_SOURCE_VALUE_KEYS = new Set([
|
|
|
336
337
|
BUILDER_CMS_BODY_SIDECARS_KEY,
|
|
337
338
|
]);
|
|
338
339
|
|
|
340
|
+
const SOURCE_VALUES_JSON_COLUMN =
|
|
341
|
+
'"content_database_source_rows"."source_values_json"';
|
|
342
|
+
|
|
343
|
+
export function sourceSnapshotValuesJsonProjectionSql(dialect: Dialect) {
|
|
344
|
+
const keys = Array.from(HEAVY_BUILDER_BODY_SOURCE_VALUE_KEYS);
|
|
345
|
+
if (dialect === "postgres") {
|
|
346
|
+
return `COALESCE((${SOURCE_VALUES_JSON_COLUMN}::jsonb${keys
|
|
347
|
+
.map((key) => ` - '${key}'`)
|
|
348
|
+
.join("")})::text, '{}')`;
|
|
349
|
+
}
|
|
350
|
+
const paths = keys.map((key) => `'$."${key}"'`);
|
|
351
|
+
return `COALESCE(json_remove(${SOURCE_VALUES_JSON_COLUMN}, ${paths.join(", ")}), '{}')`;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
function sourceSnapshotRowSelection(args: {
|
|
355
|
+
stripHeavyBuilderBodyValues: boolean;
|
|
356
|
+
}) {
|
|
357
|
+
const row = schema.contentDatabaseSourceRows;
|
|
358
|
+
return {
|
|
359
|
+
id: row.id,
|
|
360
|
+
ownerEmail: row.ownerEmail,
|
|
361
|
+
sourceId: row.sourceId,
|
|
362
|
+
databaseItemId: row.databaseItemId,
|
|
363
|
+
documentId: row.documentId,
|
|
364
|
+
sourceRowId: row.sourceRowId,
|
|
365
|
+
sourceQualifiedId: row.sourceQualifiedId,
|
|
366
|
+
sourceDisplayKey: row.sourceDisplayKey,
|
|
367
|
+
sourceValuesJson: args.stripHeavyBuilderBodyValues
|
|
368
|
+
? sql<string>`${sql.raw(
|
|
369
|
+
sourceSnapshotValuesJsonProjectionSql(getDialect()),
|
|
370
|
+
)}`
|
|
371
|
+
: row.sourceValuesJson,
|
|
372
|
+
provenance: row.provenance,
|
|
373
|
+
syncState: row.syncState,
|
|
374
|
+
freshness: row.freshness,
|
|
375
|
+
lastSyncedAt: row.lastSyncedAt,
|
|
376
|
+
lastSourceUpdatedAt: row.lastSourceUpdatedAt,
|
|
377
|
+
createdAt: row.createdAt,
|
|
378
|
+
updatedAt: row.updatedAt,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
339
382
|
export function sourceValuesForSnapshot(
|
|
340
383
|
sourceValues: Record<string, DocumentPropertyValue>,
|
|
341
384
|
options: { includeHeavyBuilderBodyValues?: boolean } = {},
|
|
@@ -2389,10 +2432,16 @@ async function readSourceSnapshotRowsOnce(args: {
|
|
|
2389
2432
|
source: ContentDatabaseSourceRowDb;
|
|
2390
2433
|
database: ContentDatabaseRow | ContentDatabase;
|
|
2391
2434
|
isBuilderSource: boolean;
|
|
2435
|
+
includeHeavyBuilderBodyValues: boolean;
|
|
2392
2436
|
}) {
|
|
2393
2437
|
const db = getDb();
|
|
2394
2438
|
const rowRows = await db
|
|
2395
|
-
.select(
|
|
2439
|
+
.select(
|
|
2440
|
+
sourceSnapshotRowSelection({
|
|
2441
|
+
stripHeavyBuilderBodyValues:
|
|
2442
|
+
args.isBuilderSource && !args.includeHeavyBuilderBodyValues,
|
|
2443
|
+
}),
|
|
2444
|
+
)
|
|
2396
2445
|
.from(schema.contentDatabaseSourceRows)
|
|
2397
2446
|
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id))
|
|
2398
2447
|
.orderBy(asc(schema.contentDatabaseSourceRows.createdAt));
|
|
@@ -2467,6 +2516,7 @@ async function sourceSnapshotConsistencyMarker(args: {
|
|
|
2467
2516
|
source: ContentDatabaseSourceRowDb;
|
|
2468
2517
|
database: ContentDatabaseRow | ContentDatabase;
|
|
2469
2518
|
isBuilderSource: boolean;
|
|
2519
|
+
includeHeavyBuilderBodyValues: boolean;
|
|
2470
2520
|
}) {
|
|
2471
2521
|
const db = getDb();
|
|
2472
2522
|
const [rows] = await db
|
|
@@ -2518,6 +2568,7 @@ async function loadSourceSnapshotRowsOptimistically(args: {
|
|
|
2518
2568
|
source: ContentDatabaseSourceRowDb;
|
|
2519
2569
|
database: ContentDatabaseRow | ContentDatabase;
|
|
2520
2570
|
isBuilderSource: boolean;
|
|
2571
|
+
includeHeavyBuilderBodyValues: boolean;
|
|
2521
2572
|
}) {
|
|
2522
2573
|
let latest: Awaited<ReturnType<typeof readSourceSnapshotRowsOnce>> | null =
|
|
2523
2574
|
null;
|
|
@@ -2616,6 +2667,7 @@ async function loadSourceSnapshot(
|
|
|
2616
2667
|
source,
|
|
2617
2668
|
database,
|
|
2618
2669
|
isBuilderSource,
|
|
2670
|
+
includeHeavyBuilderBodyValues: options.includeHeavyBuilderBodyValues,
|
|
2619
2671
|
});
|
|
2620
2672
|
const rows = rowRows.map((row) =>
|
|
2621
2673
|
serializeSourceRowRecord(row, {
|
|
@@ -2722,7 +2774,7 @@ async function loadSourceSnapshot(
|
|
|
2722
2774
|
string,
|
|
2723
2775
|
ContentDatabaseSourceBodyChange
|
|
2724
2776
|
>();
|
|
2725
|
-
if (isBuilderSource) {
|
|
2777
|
+
if (isBuilderSource && options.includeHeavyBuilderBodyValues) {
|
|
2726
2778
|
const sourceRowByDocumentId = new Map(
|
|
2727
2779
|
rowRows.map((row) => [row.documentId, row]),
|
|
2728
2780
|
);
|
|
@@ -2860,6 +2912,10 @@ async function loadSourceSnapshot(
|
|
|
2860
2912
|
typeof metadata.lastReadNextOffset === "number"
|
|
2861
2913
|
? metadata.lastReadNextOffset
|
|
2862
2914
|
: undefined,
|
|
2915
|
+
lastReadSuspiciousEmpty:
|
|
2916
|
+
typeof metadata.lastReadSuspiciousEmpty === "boolean"
|
|
2917
|
+
? metadata.lastReadSuspiciousEmpty
|
|
2918
|
+
: undefined,
|
|
2863
2919
|
sourceFetchState:
|
|
2864
2920
|
metadata.sourceFetchState === "idle" ||
|
|
2865
2921
|
metadata.sourceFetchState === "fetching" ||
|
|
@@ -3019,6 +3075,7 @@ export function serializeBuilderCmsSourceReadMetadataRecord(args: {
|
|
|
3019
3075
|
progress?: BuilderCmsReadProgress;
|
|
3020
3076
|
sourceFetchState?: "idle" | "fetching" | "error";
|
|
3021
3077
|
activeReadSourceRowIds?: string[];
|
|
3078
|
+
suspiciousEmpty?: boolean;
|
|
3022
3079
|
builderModelFields?: BuilderCmsModelFieldSummary[];
|
|
3023
3080
|
existingMetadataJson?: string | null;
|
|
3024
3081
|
}) {
|
|
@@ -3038,7 +3095,10 @@ export function serializeBuilderCmsSourceReadMetadataRecord(args: {
|
|
|
3038
3095
|
lastReadPartial: args.progress?.partial,
|
|
3039
3096
|
lastReadHasMore: args.progress?.hasMore,
|
|
3040
3097
|
lastReadNextOffset: args.progress?.nextOffset,
|
|
3041
|
-
|
|
3098
|
+
lastReadSuspiciousEmpty: args.suspiciousEmpty === true,
|
|
3099
|
+
activeReadSourceRowIds: args.suspiciousEmpty
|
|
3100
|
+
? []
|
|
3101
|
+
: args.activeReadSourceRowIds,
|
|
3042
3102
|
sourceFetchState:
|
|
3043
3103
|
args.sourceFetchState ??
|
|
3044
3104
|
(args.progress?.partial
|
|
@@ -3122,6 +3182,11 @@ export async function seedMockSourceFields(args: {
|
|
|
3122
3182
|
}) {
|
|
3123
3183
|
const db = getDb();
|
|
3124
3184
|
const isBuilder = args.sourceType === "builder-cms";
|
|
3185
|
+
const existingBuilderFieldByPropertyId = new Map(
|
|
3186
|
+
(args.existingFields ?? [])
|
|
3187
|
+
.filter((field) => isBuilder && field.propertyId)
|
|
3188
|
+
.map((field) => [field.propertyId!, field]),
|
|
3189
|
+
);
|
|
3125
3190
|
const rows = [
|
|
3126
3191
|
{
|
|
3127
3192
|
id: crypto.randomUUID(),
|
|
@@ -3224,10 +3289,12 @@ export async function seedMockSourceFields(args: {
|
|
|
3224
3289
|
propertyId: property.definition.id,
|
|
3225
3290
|
localFieldKey: property.definition.id,
|
|
3226
3291
|
sourceFieldKey: isBuilder
|
|
3227
|
-
?
|
|
3292
|
+
? (existingBuilderFieldByPropertyId.get(property.definition.id)
|
|
3293
|
+
?.sourceFieldKey ??
|
|
3294
|
+
builderCmsSourceFieldKey(
|
|
3228
3295
|
property.definition.id,
|
|
3229
3296
|
property.definition.name,
|
|
3230
|
-
)
|
|
3297
|
+
))
|
|
3231
3298
|
: `fields.${slugifySourceField(property.definition.name)}`,
|
|
3232
3299
|
sourceFieldLabel: property.definition.name,
|
|
3233
3300
|
sourceFieldType: property.definition.type,
|
|
@@ -3259,15 +3326,19 @@ export async function seedMockSourceFields(args: {
|
|
|
3259
3326
|
updatedAt: args.now,
|
|
3260
3327
|
})),
|
|
3261
3328
|
];
|
|
3329
|
+
const sourceFieldIdentityKey = (sourceFieldKey: string) => {
|
|
3330
|
+
const trimmed = sourceFieldKey.trim();
|
|
3331
|
+
return isBuilder ? trimmed : trimmed.toLowerCase();
|
|
3332
|
+
};
|
|
3262
3333
|
if (isBuilder) {
|
|
3263
3334
|
const existingSourceFieldKeys = new Set(
|
|
3264
|
-
rows.map((row) => row.sourceFieldKey
|
|
3335
|
+
rows.map((row) => sourceFieldIdentityKey(row.sourceFieldKey)),
|
|
3265
3336
|
);
|
|
3266
3337
|
for (const field of args.builderModelFields ?? []) {
|
|
3267
3338
|
const fieldName = field.name.trim();
|
|
3268
3339
|
if (!fieldName) continue;
|
|
3269
3340
|
const sourceFieldKey = `data.${fieldName}`;
|
|
3270
|
-
const normalizedKey = sourceFieldKey
|
|
3341
|
+
const normalizedKey = sourceFieldIdentityKey(sourceFieldKey);
|
|
3271
3342
|
if (existingSourceFieldKeys.has(normalizedKey)) continue;
|
|
3272
3343
|
existingSourceFieldKeys.add(normalizedKey);
|
|
3273
3344
|
rows.push({
|
|
@@ -3292,7 +3363,7 @@ export async function seedMockSourceFields(args: {
|
|
|
3292
3363
|
for (const entry of args.builderSampleEntries ?? []) {
|
|
3293
3364
|
for (const sourceFieldKey of Object.keys(entry.sourceValues)) {
|
|
3294
3365
|
if (!sourceFieldKey.startsWith("data.")) continue;
|
|
3295
|
-
const normalizedKey = sourceFieldKey
|
|
3366
|
+
const normalizedKey = sourceFieldIdentityKey(sourceFieldKey);
|
|
3296
3367
|
if (existingSourceFieldKeys.has(normalizedKey)) continue;
|
|
3297
3368
|
existingSourceFieldKeys.add(normalizedKey);
|
|
3298
3369
|
const value = entry.sourceValues[sourceFieldKey];
|
|
@@ -3329,13 +3400,13 @@ export async function seedMockSourceFields(args: {
|
|
|
3329
3400
|
|
|
3330
3401
|
const existingFieldBySourceKey = new Map(
|
|
3331
3402
|
(args.existingFields ?? []).map((field) => [
|
|
3332
|
-
field.sourceFieldKey
|
|
3403
|
+
sourceFieldIdentityKey(field.sourceFieldKey),
|
|
3333
3404
|
field,
|
|
3334
3405
|
]),
|
|
3335
3406
|
);
|
|
3336
3407
|
const mergedRows = rows.map((row) => {
|
|
3337
3408
|
const existing = existingFieldBySourceKey.get(
|
|
3338
|
-
row.sourceFieldKey
|
|
3409
|
+
sourceFieldIdentityKey(row.sourceFieldKey),
|
|
3339
3410
|
);
|
|
3340
3411
|
if (!existing) return row;
|
|
3341
3412
|
return {
|
|
@@ -4152,8 +4223,33 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
4152
4223
|
sourceMetadata.lastReadNextOffset > 0
|
|
4153
4224
|
? sourceMetadata.lastReadNextOffset
|
|
4154
4225
|
: 0;
|
|
4226
|
+
const existingFields = await db
|
|
4227
|
+
.select()
|
|
4228
|
+
.from(schema.contentDatabaseSourceFields)
|
|
4229
|
+
.where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
|
|
4230
|
+
let builderModelFields: BuilderCmsModelFieldSummary[] | undefined;
|
|
4231
|
+
let builderModelFieldsReadFailed = false;
|
|
4232
|
+
try {
|
|
4233
|
+
builderModelFields = await readBuilderCmsModelFields({
|
|
4234
|
+
model: args.source.sourceTable,
|
|
4235
|
+
});
|
|
4236
|
+
} catch (error) {
|
|
4237
|
+
builderModelFieldsReadFailed = true;
|
|
4238
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4239
|
+
console.warn(
|
|
4240
|
+
`[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`,
|
|
4241
|
+
);
|
|
4242
|
+
}
|
|
4243
|
+
const projectionModelFields =
|
|
4244
|
+
builderModelFields && builderModelFields.length > 0
|
|
4245
|
+
? builderModelFields
|
|
4246
|
+
: (sourceMetadata.builderModelFields ?? []);
|
|
4155
4247
|
const builderRead = await readBuilderCmsContentEntries({
|
|
4156
4248
|
model: args.source.sourceTable,
|
|
4249
|
+
fieldPaths: [
|
|
4250
|
+
...existingFields.map((field) => field.sourceFieldKey),
|
|
4251
|
+
...projectionModelFields.map((field) => `data.${field.name}`),
|
|
4252
|
+
],
|
|
4157
4253
|
maxPages: args.runFullRefresh
|
|
4158
4254
|
? undefined
|
|
4159
4255
|
: BUILDER_CMS_REFRESH_INITIAL_PAGES,
|
|
@@ -4170,15 +4266,41 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
4170
4266
|
.select()
|
|
4171
4267
|
.from(schema.contentDatabaseSourceRows)
|
|
4172
4268
|
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
|
|
4269
|
+
const readStartOffset = builderRead.progress?.startOffset ?? 0;
|
|
4270
|
+
const activeReadSourceRowIdSet = new Set(activeReadSourceRowIds);
|
|
4271
|
+
const suspiciousEmptyRead =
|
|
4272
|
+
builderRead.state === "live" &&
|
|
4273
|
+
builderRead.entries.length === 0 &&
|
|
4274
|
+
existingRows.length > 0 &&
|
|
4275
|
+
(readStartOffset === 0 ||
|
|
4276
|
+
existingRows.some(
|
|
4277
|
+
(row) => !activeReadSourceRowIdSet.has(row.sourceRowId),
|
|
4278
|
+
));
|
|
4279
|
+
if (suspiciousEmptyRead) {
|
|
4280
|
+
const message =
|
|
4281
|
+
"Builder CMS returned no entries for a source with existing rows. The previous snapshot was preserved; retry the refresh before treating the source as empty.";
|
|
4282
|
+
await updateBuilderCmsSourceReadMetadata({
|
|
4283
|
+
sourceId: args.source.id,
|
|
4284
|
+
sourceTable: args.source.sourceTable,
|
|
4285
|
+
readState: builderRead.state,
|
|
4286
|
+
entryCount: builderRead.entries.length,
|
|
4287
|
+
matchedRowCount: 0,
|
|
4288
|
+
fetchedAt: builderRead.fetchedAt,
|
|
4289
|
+
now: args.now,
|
|
4290
|
+
message,
|
|
4291
|
+
builderModelFields,
|
|
4292
|
+
progress: builderRead.progress,
|
|
4293
|
+
sourceFetchState: "error",
|
|
4294
|
+
syncState: "error",
|
|
4295
|
+
suspiciousEmpty: true,
|
|
4296
|
+
});
|
|
4297
|
+
return;
|
|
4298
|
+
}
|
|
4173
4299
|
await enqueueEmptyHydratedBuilderBodiesFromStoredRows({
|
|
4174
4300
|
source: args.source,
|
|
4175
4301
|
now: args.now,
|
|
4176
4302
|
});
|
|
4177
4303
|
let importedEntriesByDocumentId = new Map<string, BuilderCmsSourceEntry>();
|
|
4178
|
-
const existingFields = await db
|
|
4179
|
-
.select()
|
|
4180
|
-
.from(schema.contentDatabaseSourceFields)
|
|
4181
|
-
.where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id));
|
|
4182
4304
|
if (builderRead.state === "live") {
|
|
4183
4305
|
const importResult = await importBuilderCmsEntriesAsDatabaseItems({
|
|
4184
4306
|
database: args.database,
|
|
@@ -4196,19 +4318,6 @@ export async function resyncBuilderCmsSourceSnapshot(args: {
|
|
|
4196
4318
|
.where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id));
|
|
4197
4319
|
}
|
|
4198
4320
|
}
|
|
4199
|
-
let builderModelFields: BuilderCmsModelFieldSummary[] | undefined;
|
|
4200
|
-
let builderModelFieldsReadFailed = false;
|
|
4201
|
-
try {
|
|
4202
|
-
builderModelFields = await readBuilderCmsModelFields({
|
|
4203
|
-
model: args.source.sourceTable,
|
|
4204
|
-
});
|
|
4205
|
-
} catch (error) {
|
|
4206
|
-
builderModelFieldsReadFailed = true;
|
|
4207
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
4208
|
-
console.warn(
|
|
4209
|
-
`[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`,
|
|
4210
|
-
);
|
|
4211
|
-
}
|
|
4212
4321
|
const builderEntriesByDocumentId =
|
|
4213
4322
|
builderRead.state === "live"
|
|
4214
4323
|
? mapBuilderCmsEntriesToLocalItems({
|
|
@@ -4797,12 +4906,14 @@ export async function updateBuilderCmsSourceReadMetadata(args: {
|
|
|
4797
4906
|
activeReadSourceRowIds?: string[];
|
|
4798
4907
|
syncState?: ContentDatabaseSourceSyncState;
|
|
4799
4908
|
builderModelFields?: BuilderCmsModelFieldSummary[];
|
|
4909
|
+
suspiciousEmpty?: boolean;
|
|
4800
4910
|
}) {
|
|
4801
4911
|
const db = getDb();
|
|
4802
4912
|
const [currentSource] = await db
|
|
4803
4913
|
.select({
|
|
4804
4914
|
capabilitiesJson: schema.contentDatabaseSources.capabilitiesJson,
|
|
4805
4915
|
metadataJson: schema.contentDatabaseSources.metadataJson,
|
|
4916
|
+
lastSourceUpdatedAt: schema.contentDatabaseSources.lastSourceUpdatedAt,
|
|
4806
4917
|
})
|
|
4807
4918
|
.from(schema.contentDatabaseSources)
|
|
4808
4919
|
.where(eq(schema.contentDatabaseSources.id, args.sourceId))
|
|
@@ -4820,6 +4931,7 @@ export async function updateBuilderCmsSourceReadMetadata(args: {
|
|
|
4820
4931
|
progress: args.progress,
|
|
4821
4932
|
sourceFetchState: args.sourceFetchState,
|
|
4822
4933
|
activeReadSourceRowIds: args.activeReadSourceRowIds,
|
|
4934
|
+
suspiciousEmpty: args.suspiciousEmpty,
|
|
4823
4935
|
builderModelFields: args.builderModelFields,
|
|
4824
4936
|
existingMetadataJson: currentSource?.metadataJson,
|
|
4825
4937
|
}),
|
|
@@ -4829,14 +4941,21 @@ export async function updateBuilderCmsSourceReadMetadata(args: {
|
|
|
4829
4941
|
.set({
|
|
4830
4942
|
syncState: args.syncState ?? "linked",
|
|
4831
4943
|
freshness:
|
|
4832
|
-
args.readState === "error" ||
|
|
4944
|
+
args.readState === "error" ||
|
|
4945
|
+
args.progress?.partial ||
|
|
4946
|
+
args.suspiciousEmpty
|
|
4833
4947
|
? "stale"
|
|
4834
4948
|
: "fresh",
|
|
4835
4949
|
capabilitiesJson: nextJson.capabilitiesJson,
|
|
4836
4950
|
metadataJson: nextJson.metadataJson,
|
|
4837
4951
|
lastRefreshedAt: args.now,
|
|
4838
|
-
lastSourceUpdatedAt: args.
|
|
4839
|
-
|
|
4952
|
+
lastSourceUpdatedAt: args.suspiciousEmpty
|
|
4953
|
+
? (currentSource?.lastSourceUpdatedAt ?? null)
|
|
4954
|
+
: args.fetchedAt,
|
|
4955
|
+
lastError:
|
|
4956
|
+
args.readState === "error" || args.suspiciousEmpty
|
|
4957
|
+
? args.message
|
|
4958
|
+
: null,
|
|
4840
4959
|
updatedAt: args.now,
|
|
4841
4960
|
})
|
|
4842
4961
|
.where(eq(schema.contentDatabaseSources.id, args.sourceId));
|
|
@@ -201,10 +201,7 @@ export function sourceFieldPropertyOptions(args: {
|
|
|
201
201
|
}
|
|
202
202
|
|
|
203
203
|
function builderFieldNameForSourceKey(sourceFieldKey: string) {
|
|
204
|
-
return sourceFieldKey
|
|
205
|
-
.replace(/^data\./, "")
|
|
206
|
-
.trim()
|
|
207
|
-
.toLowerCase();
|
|
204
|
+
return sourceFieldKey.replace(/^data\./, "").trim();
|
|
208
205
|
}
|
|
209
206
|
|
|
210
207
|
function builderModelFieldsFromMetadata(
|
|
@@ -231,15 +228,15 @@ function builderModelFieldsFromMetadata(
|
|
|
231
228
|
}
|
|
232
229
|
}
|
|
233
230
|
|
|
234
|
-
function builderMetadataForSourceField(args: {
|
|
231
|
+
export function builderMetadataForSourceField(args: {
|
|
235
232
|
sourceFieldKey: string;
|
|
236
233
|
sourceMetadataJson: string | null | undefined;
|
|
237
234
|
}) {
|
|
238
235
|
const fieldName = builderFieldNameForSourceKey(args.sourceFieldKey);
|
|
239
|
-
const sourceFieldKey = args.sourceFieldKey.trim()
|
|
236
|
+
const sourceFieldKey = args.sourceFieldKey.trim();
|
|
240
237
|
return (
|
|
241
238
|
builderModelFieldsFromMetadata(args.sourceMetadataJson).find((field) => {
|
|
242
|
-
const name = field.name.trim()
|
|
239
|
+
const name = field.name.trim();
|
|
243
240
|
return name === fieldName || `data.${name}` === sourceFieldKey;
|
|
244
241
|
}) ?? null
|
|
245
242
|
);
|
|
@@ -48,13 +48,41 @@ const BUILDER_CMS_ATTACH_INITIAL_PAGES = 1;
|
|
|
48
48
|
export async function readInitialBuilderCmsAttachEntries(
|
|
49
49
|
sourceTable: string,
|
|
50
50
|
readEntries: typeof readBuilderCmsContentEntries = readBuilderCmsContentEntries,
|
|
51
|
+
fieldPaths: readonly string[] = [],
|
|
51
52
|
) {
|
|
52
53
|
return readEntries({
|
|
53
54
|
model: sourceTable,
|
|
55
|
+
fieldPaths,
|
|
54
56
|
maxPages: BUILDER_CMS_ATTACH_INITIAL_PAGES,
|
|
55
57
|
});
|
|
56
58
|
}
|
|
57
59
|
|
|
60
|
+
export async function readInitialBuilderCmsAttachSource(
|
|
61
|
+
sourceTable: string,
|
|
62
|
+
dependencies: {
|
|
63
|
+
readModelFields?: typeof readBuilderCmsModelFields;
|
|
64
|
+
readEntries?: typeof readBuilderCmsContentEntries;
|
|
65
|
+
} = {},
|
|
66
|
+
) {
|
|
67
|
+
const readModelFields =
|
|
68
|
+
dependencies.readModelFields ?? readBuilderCmsModelFields;
|
|
69
|
+
const readEntries = dependencies.readEntries ?? readBuilderCmsContentEntries;
|
|
70
|
+
let modelFields: BuilderCmsModelFieldSummary[] = [];
|
|
71
|
+
let modelFieldsError: unknown = null;
|
|
72
|
+
try {
|
|
73
|
+
modelFields = await readModelFields({ model: sourceTable });
|
|
74
|
+
} catch (error) {
|
|
75
|
+
modelFieldsError = error;
|
|
76
|
+
}
|
|
77
|
+
const read = await readInitialBuilderCmsAttachEntries(
|
|
78
|
+
sourceTable,
|
|
79
|
+
readEntries,
|
|
80
|
+
modelFields.map((field) => `data.${field.name}`),
|
|
81
|
+
);
|
|
82
|
+
if (modelFieldsError) throw modelFieldsError;
|
|
83
|
+
return { read, modelFields };
|
|
84
|
+
}
|
|
85
|
+
|
|
58
86
|
function builderReadHasMore(read: BuilderCmsReadResult | null | undefined) {
|
|
59
87
|
return read?.state === "live" && read.progress?.hasMore === true;
|
|
60
88
|
}
|
|
@@ -238,9 +266,10 @@ export default defineAction({
|
|
|
238
266
|
let modelFields: BuilderCmsModelFieldSummary[];
|
|
239
267
|
let builderRead: BuilderCmsReadResult | null = null;
|
|
240
268
|
if (sourceType === "builder-cms") {
|
|
241
|
-
|
|
269
|
+
const initial = await readInitialBuilderCmsAttachSource(sourceTable);
|
|
270
|
+
modelFields = initial.modelFields;
|
|
271
|
+
builderRead = initial.read;
|
|
242
272
|
entries = builderRead.state === "live" ? builderRead.entries : [];
|
|
243
|
-
modelFields = await readBuilderCmsModelFields({ model: sourceTable });
|
|
244
273
|
} else if (sourceType === "local-table") {
|
|
245
274
|
// sourceTable carries the target database id for a local-table source.
|
|
246
275
|
({ entries, modelFields } = await readLocalTableEntries(sourceTable, {
|
|
@@ -327,13 +356,12 @@ export default defineAction({
|
|
|
327
356
|
if (await databaseSourceExistsForTable(database.id, sourceTable)) {
|
|
328
357
|
throw new Error(`"${sourceTable}" is already attached as a source.`);
|
|
329
358
|
}
|
|
330
|
-
const
|
|
331
|
-
await
|
|
359
|
+
const additionalInitial =
|
|
360
|
+
await readInitialBuilderCmsAttachSource(sourceTable);
|
|
361
|
+
const additionalModelFields = additionalInitial.modelFields;
|
|
362
|
+
const additionalRead = additionalInitial.read;
|
|
332
363
|
const additionalEntries =
|
|
333
364
|
additionalRead.state === "live" ? additionalRead.entries : [];
|
|
334
|
-
const additionalModelFields = await readBuilderCmsModelFields({
|
|
335
|
-
model: sourceTable,
|
|
336
|
-
});
|
|
337
365
|
const additionalSourceId = await insertSecondarySource({
|
|
338
366
|
database,
|
|
339
367
|
sourceType,
|
|
@@ -436,6 +464,10 @@ export default defineAction({
|
|
|
436
464
|
const existingSourceRows = existingSource
|
|
437
465
|
? await getSourceRows(existingSource.id)
|
|
438
466
|
: [];
|
|
467
|
+
const builderInitial =
|
|
468
|
+
sourceType === "builder-cms"
|
|
469
|
+
? await readInitialBuilderCmsAttachSource(sourceTable)
|
|
470
|
+
: null;
|
|
439
471
|
const sourceId = await replaceSourceMetadata({
|
|
440
472
|
database,
|
|
441
473
|
source: existingSource,
|
|
@@ -444,18 +476,10 @@ export default defineAction({
|
|
|
444
476
|
sourceTable,
|
|
445
477
|
now,
|
|
446
478
|
});
|
|
447
|
-
const
|
|
448
|
-
|
|
449
|
-
? await readInitialBuilderCmsAttachEntries(sourceTable)
|
|
450
|
-
: null;
|
|
479
|
+
const builderModelFields = builderInitial?.modelFields ?? [];
|
|
480
|
+
const builderRead = builderInitial?.read ?? null;
|
|
451
481
|
const builderEntries =
|
|
452
482
|
builderRead?.state === "live" ? builderRead.entries : [];
|
|
453
|
-
const builderModelFields =
|
|
454
|
-
sourceType === "builder-cms"
|
|
455
|
-
? await readBuilderCmsModelFields({
|
|
456
|
-
model: sourceTable,
|
|
457
|
-
})
|
|
458
|
-
: [];
|
|
459
483
|
let importedEntriesByDocumentId = new Map<string, BuilderCmsSourceEntry>();
|
|
460
484
|
if (builderRead?.state === "live") {
|
|
461
485
|
const importResult = await importBuilderCmsEntriesAsDatabaseItems({
|