@agent-native/core-corpus 0.210.1-nightly-20261020124228 → 0.210.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/analytics/.agents/skills/dashboard-management/SKILL.md +19 -0
- package/corpus/templates/analytics/CHANGELOG.md +14 -0
- package/corpus/templates/analytics/actions/builder-blog-articles.ts +184 -40
- package/corpus/templates/analytics/actions/configure-content-collection.ts +74 -0
- package/corpus/templates/analytics/actions/get-content-collection.ts +33 -0
- package/corpus/templates/analytics/actions/get-first-party-analytics-backend.ts +3 -0
- package/corpus/templates/analytics/actions/query-dbt-semantic-metric.ts +15 -12
- package/corpus/templates/analytics/actions/run-content-collection.ts +50 -0
- package/corpus/templates/analytics/changelog/2026-10-15-analytics-can-collect-daily-ga4-pageviews-and-search-console.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-15-analytics-can-collect-your-blog-article-list-once-a-day-into.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-15-the-blog-article-tool-now-returns-each-article-s-author-and-.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-20-blog-article-credits-include-direct-and-wrapped-author-refer.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-20-content-collection-recovers-abandoned-runs-avoids-duplicate-.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-20-daily-content-metrics-recover-collection-gaps-and-keep-omitt.md +6 -0
- package/corpus/templates/analytics/changelog/2026-10-20-fixed-analytics-dbt-metric-lookups-that-returned-a-bad-gatew.md +6 -0
- package/corpus/templates/analytics/netlify.toml +3 -0
- package/corpus/templates/analytics/scripts/emit-netlify-dashboard-report-cron.ts +60 -17
- package/corpus/templates/analytics/server/db/schema.ts +104 -0
- package/corpus/templates/analytics/server/jobs/content-metrics-collection.ts +20 -0
- package/corpus/templates/analytics/server/lib/content-metrics/collect.ts +141 -0
- package/corpus/templates/analytics/server/lib/content-metrics/config.ts +69 -0
- package/corpus/templates/analytics/server/lib/content-metrics/migration.ts +67 -0
- package/corpus/templates/analytics/server/lib/content-metrics/status.ts +96 -0
- package/corpus/templates/analytics/server/lib/content-metrics/step.ts +62 -0
- package/corpus/templates/analytics/server/lib/content-metrics/steps/builder-blog.ts +67 -0
- package/corpus/templates/analytics/server/lib/content-metrics/steps/daily-metrics.ts +145 -0
- package/corpus/templates/analytics/server/lib/content-metrics/steps/ga4.ts +115 -0
- package/corpus/templates/analytics/server/lib/content-metrics/steps/search-console.ts +176 -0
- package/corpus/templates/analytics/server/lib/content-metrics/store.ts +502 -0
- package/corpus/templates/analytics/server/lib/first-party-analytics.ts +44 -13
- package/corpus/templates/analytics/server/lib/google-analytics.ts +121 -9
- package/corpus/templates/analytics/server/lib/search-console.ts +105 -0
- package/corpus/templates/analytics/server/plugins/content-metrics-collection-jobs.ts +43 -0
- package/corpus/templates/analytics/server/plugins/db.ts +20 -8
- package/corpus/templates/analytics/server/routes/api/content-metrics/collect.post.ts +19 -0
- package/corpus/templates/brain/CHANGELOG.md +6 -0
- package/corpus/templates/brain/actions/list-sources.ts +105 -39
- package/corpus/templates/brain/changelog/2026-10-19-brain-source-list-uses-fewer-database-queries.md +6 -0
- package/corpus/templates/brain/server/lib/audiences.ts +21 -8
- package/corpus/templates/brain/server/lib/zoom.ts +56 -24
- package/corpus/templates/clips/CHANGELOG.md +1 -0
- package/corpus/templates/content/CHANGELOG.md +7 -0
- package/corpus/templates/design/CHANGELOG.md +60 -0
- package/corpus/templates/design/app/components/editor/PromptDialog.tsx +6 -1
- package/corpus/templates/design/app/lib/prompt-image-optimization.ts +1 -1
- package/corpus/templates/design/app/pages/Index.tsx +79 -45
- package/corpus/templates/design/app/pages/design-editor/domains/use-editor-export-and-handoff.ts +1 -1
- package/corpus/templates/design/changelog/2026-10-20-high-resolution-reference-images-are-resized-in-prompts.md +6 -0
- package/corpus/templates/design/changelog/2026-10-20-public-design-viewers-no-longer-see-repeated-png-export-erro.md +6 -0
- package/corpus/templates/design/changelog/2026-10-20-skip-prompt-moves-beside-suggestions.md +6 -0
- package/corpus/templates/design/changelog/2026-10-20-the-design-home-chat-keeps-its-model-picker-visible-while-re.md +6 -0
- package/corpus/templates/slides/CHANGELOG.md +6 -0
- package/corpus/templates/slides/actions/_uploaded-files.ts +128 -13
- package/corpus/templates/slides/actions/import-docx.ts +6 -2
- package/corpus/templates/slides/actions/import-file.ts +12 -2
- package/corpus/templates/slides/actions/import-pptx.ts +6 -2
- package/corpus/templates/slides/changelog/2026-10-20-slides-handles-chat-attachments-more-reliably-including-docu.md +6 -0
- package/corpus/templates/slides/server/lib/chat-attachments.ts +27 -7
- package/package.json +1 -1
- package/corpus/templates/analytics/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/assets/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/brain/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/calendar/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/chat/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/clips/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/content/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/crm/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/design/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/design/.generated/bridge/native-builtin-metadata.generated.ts +0 -301
- package/corpus/templates/design/.generated/bridge/native-worker-source.generated.ts +0 -6
- package/corpus/templates/dispatch/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/forms/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/mail/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/plan/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
- package/corpus/templates/slides/.deploy-tmp/agent-native-provided-plugins.mjs +0 -10
|
@@ -50,6 +50,10 @@ const RETENTION_SCHEDULED_NAME = "session-replay-retention-cron";
|
|
|
50
50
|
const RETENTION_WORKER_NAME = "session-replay-retention-sweep-background";
|
|
51
51
|
const RETENTION_ROUTE_PATH = "/api/session-replay/retention";
|
|
52
52
|
const RETENTION_SCHEDULE = "0 3 * * *";
|
|
53
|
+
const CONTENT_METRICS_SCHEDULED_NAME = "content-metrics-cron";
|
|
54
|
+
const CONTENT_METRICS_WORKER_NAME = "content-metrics-collection-background";
|
|
55
|
+
const CONTENT_METRICS_ROUTE_PATH = "/api/content-metrics/collect";
|
|
56
|
+
const CONTENT_METRICS_SCHEDULE = "0 7 * * *";
|
|
53
57
|
|
|
54
58
|
function ensureDir(dir: string) {
|
|
55
59
|
mkdirSync(dir, { recursive: true });
|
|
@@ -619,12 +623,45 @@ export const config = {
|
|
|
619
623
|
writeWorkerEntry(dest, BIGQUERY_WORKER_NAME, source);
|
|
620
624
|
}
|
|
621
625
|
|
|
622
|
-
|
|
623
|
-
|
|
626
|
+
type DailyJobSpec = {
|
|
627
|
+
scheduledName: string;
|
|
628
|
+
workerName: string;
|
|
629
|
+
routePath: string;
|
|
630
|
+
schedule: string;
|
|
631
|
+
header: string;
|
|
632
|
+
runtimeFlag: string;
|
|
633
|
+
label: string;
|
|
634
|
+
logPrefix: string;
|
|
635
|
+
};
|
|
636
|
+
|
|
637
|
+
const RETENTION_JOB: DailyJobSpec = {
|
|
638
|
+
scheduledName: RETENTION_SCHEDULED_NAME,
|
|
639
|
+
workerName: RETENTION_WORKER_NAME,
|
|
640
|
+
routePath: RETENTION_ROUTE_PATH,
|
|
641
|
+
schedule: RETENTION_SCHEDULE,
|
|
642
|
+
header: "x-agent-native-session-replay-retention-cron",
|
|
643
|
+
runtimeFlag: "__AGENT_NATIVE_SESSION_REPLAY_RETENTION_SCHEDULED_RUNTIME__",
|
|
644
|
+
label: "session replay retention",
|
|
645
|
+
logPrefix: "session-replay-retention-cron",
|
|
646
|
+
};
|
|
647
|
+
|
|
648
|
+
const CONTENT_METRICS_JOB: DailyJobSpec = {
|
|
649
|
+
scheduledName: CONTENT_METRICS_SCHEDULED_NAME,
|
|
650
|
+
workerName: CONTENT_METRICS_WORKER_NAME,
|
|
651
|
+
routePath: CONTENT_METRICS_ROUTE_PATH,
|
|
652
|
+
schedule: CONTENT_METRICS_SCHEDULE,
|
|
653
|
+
header: "x-agent-native-content-metrics-cron",
|
|
654
|
+
runtimeFlag: "__AGENT_NATIVE_CONTENT_METRICS_SCHEDULED_RUNTIME__",
|
|
655
|
+
label: "content metrics collection",
|
|
656
|
+
logPrefix: "content-metrics-cron",
|
|
657
|
+
};
|
|
658
|
+
|
|
659
|
+
function emitDailyScheduledTrigger(job: DailyJobSpec, token: string) {
|
|
660
|
+
const dest = path.join(FUNCTIONS_DIR, job.scheduledName);
|
|
624
661
|
rmSync(dest, { recursive: true, force: true });
|
|
625
662
|
ensureDir(dest);
|
|
626
663
|
|
|
627
|
-
const source = `const WORKER_PATH = "/.netlify/functions/${
|
|
664
|
+
const source = `const WORKER_PATH = "/.netlify/functions/${job.workerName}";
|
|
628
665
|
const CRON_TOKEN = ${JSON.stringify(token)};
|
|
629
666
|
|
|
630
667
|
function siteOrigin(request) {
|
|
@@ -640,14 +677,14 @@ export default async function handler(request) {
|
|
|
640
677
|
method: "POST",
|
|
641
678
|
headers: {
|
|
642
679
|
"content-type": "application/json",
|
|
643
|
-
"
|
|
680
|
+
"${job.header}": CRON_TOKEN,
|
|
644
681
|
},
|
|
645
682
|
body: JSON.stringify({ scheduled: true }),
|
|
646
683
|
});
|
|
647
684
|
|
|
648
685
|
if (!response.ok && response.status !== 202) {
|
|
649
686
|
console.error(
|
|
650
|
-
"[
|
|
687
|
+
"[${job.logPrefix}] Background trigger failed:",
|
|
651
688
|
response.status,
|
|
652
689
|
// coercion-ok: the non-2xx status remains the failure signal; the body is diagnostics only.
|
|
653
690
|
await response.text().catch(() => ""),
|
|
@@ -656,27 +693,27 @@ export default async function handler(request) {
|
|
|
656
693
|
}
|
|
657
694
|
|
|
658
695
|
export const config = {
|
|
659
|
-
name: "
|
|
696
|
+
name: "${job.label} cron trigger",
|
|
660
697
|
generator: "agent-native analytics build",
|
|
661
|
-
schedule: ${JSON.stringify(
|
|
698
|
+
schedule: ${JSON.stringify(job.schedule)},
|
|
662
699
|
};
|
|
663
700
|
`;
|
|
664
701
|
|
|
665
|
-
writeFileSync(path.join(dest, `${
|
|
702
|
+
writeFileSync(path.join(dest, `${job.scheduledName}.mjs`), source);
|
|
666
703
|
}
|
|
667
704
|
|
|
668
|
-
function
|
|
669
|
-
const dest = path.join(FUNCTIONS_DIR,
|
|
705
|
+
function emitDailyBackgroundWorker(job: DailyJobSpec, token: string) {
|
|
706
|
+
const dest = path.join(FUNCTIONS_DIR, job.workerName);
|
|
670
707
|
rmSync(dest, { recursive: true, force: true });
|
|
671
708
|
cloneServerBundleForFunction(SERVER_DIR, dest);
|
|
672
709
|
rmSync(path.join(dest, "server.mjs"), { force: true });
|
|
673
710
|
|
|
674
711
|
const source = `globalThis.__AGENT_NATIVE_BACKGROUND_RUNTIME__ = true;
|
|
675
712
|
globalThis.__AGENT_NATIVE_LOW_CONNECTION_BACKGROUND_RUNTIME__ = true;
|
|
676
|
-
globalThis.
|
|
713
|
+
globalThis.${job.runtimeFlag} = true;
|
|
677
714
|
|
|
678
715
|
const CRON_TOKEN = ${JSON.stringify(token)};
|
|
679
|
-
const ROUTE_PATH = ${JSON.stringify(
|
|
716
|
+
const ROUTE_PATH = ${JSON.stringify(job.routePath)};
|
|
680
717
|
let cachedHandler;
|
|
681
718
|
|
|
682
719
|
function timingSafeEquals(a, b) {
|
|
@@ -689,7 +726,7 @@ function timingSafeEquals(a, b) {
|
|
|
689
726
|
}
|
|
690
727
|
|
|
691
728
|
export default async function handler(request, context) {
|
|
692
|
-
const token = request.headers.get("
|
|
729
|
+
const token = request.headers.get("${job.header}") || "";
|
|
693
730
|
if (!timingSafeEquals(token, CRON_TOKEN)) {
|
|
694
731
|
return new Response("Unauthorized", { status: 401 });
|
|
695
732
|
}
|
|
@@ -709,7 +746,7 @@ export default async function handler(request, context) {
|
|
|
709
746
|
}
|
|
710
747
|
|
|
711
748
|
export const config = {
|
|
712
|
-
name: "
|
|
749
|
+
name: "${job.label} background sweep",
|
|
713
750
|
generator: "agent-native analytics build",
|
|
714
751
|
background: true,
|
|
715
752
|
nodeBundler: "none",
|
|
@@ -718,7 +755,7 @@ export const config = {
|
|
|
718
755
|
};
|
|
719
756
|
`;
|
|
720
757
|
|
|
721
|
-
writeWorkerEntry(dest,
|
|
758
|
+
writeWorkerEntry(dest, job.workerName, source);
|
|
722
759
|
}
|
|
723
760
|
|
|
724
761
|
function isDirectRun(): boolean {
|
|
@@ -743,6 +780,7 @@ function main(): void {
|
|
|
743
780
|
const rollupToken = randomBytes(32).toString("hex");
|
|
744
781
|
const bigQueryToken = randomBytes(32).toString("hex");
|
|
745
782
|
const retentionToken = randomBytes(32).toString("hex");
|
|
783
|
+
const contentMetricsToken = randomBytes(32).toString("hex");
|
|
746
784
|
emitScheduledTrigger(token);
|
|
747
785
|
emitBackgroundWorker(token);
|
|
748
786
|
emitAlertScheduledTrigger(alertToken);
|
|
@@ -753,8 +791,10 @@ function main(): void {
|
|
|
753
791
|
emitRollupBackgroundWorker(rollupToken);
|
|
754
792
|
emitBigQueryScheduledTrigger(bigQueryToken);
|
|
755
793
|
emitBigQueryBackgroundWorker(bigQueryToken);
|
|
756
|
-
|
|
757
|
-
|
|
794
|
+
emitDailyScheduledTrigger(RETENTION_JOB, retentionToken);
|
|
795
|
+
emitDailyBackgroundWorker(RETENTION_JOB, retentionToken);
|
|
796
|
+
emitDailyScheduledTrigger(CONTENT_METRICS_JOB, contentMetricsToken);
|
|
797
|
+
emitDailyBackgroundWorker(CONTENT_METRICS_JOB, contentMetricsToken);
|
|
758
798
|
console.log(
|
|
759
799
|
`[dashboard-report-cron] Emitted Netlify scheduled trigger "${SCHEDULED_NAME}" (${SCHEDULE}) and background worker "${WORKER_NAME}".`,
|
|
760
800
|
);
|
|
@@ -773,6 +813,9 @@ function main(): void {
|
|
|
773
813
|
console.log(
|
|
774
814
|
`[session-replay-retention-cron] Emitted Netlify scheduled trigger "${RETENTION_SCHEDULED_NAME}" (${RETENTION_SCHEDULE}) and background worker "${RETENTION_WORKER_NAME}".`,
|
|
775
815
|
);
|
|
816
|
+
console.log(
|
|
817
|
+
`[content-metrics-cron] Emitted Netlify scheduled trigger "${CONTENT_METRICS_SCHEDULED_NAME}" (${CONTENT_METRICS_SCHEDULE}) and background worker "${CONTENT_METRICS_WORKER_NAME}".`,
|
|
818
|
+
);
|
|
776
819
|
}
|
|
777
820
|
|
|
778
821
|
if (isDirectRun()) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
bigint,
|
|
2
3
|
table,
|
|
3
4
|
text,
|
|
4
5
|
integer,
|
|
@@ -1047,3 +1048,106 @@ export const sessionReplayIngests = table(
|
|
|
1047
1048
|
),
|
|
1048
1049
|
}),
|
|
1049
1050
|
);
|
|
1051
|
+
|
|
1052
|
+
// Content view: one tenant's collection settings, read by the daily job. The
|
|
1053
|
+
// stored owner is the identity whose credentials each step resolves.
|
|
1054
|
+
export const analyticsContentCollections = table(
|
|
1055
|
+
"analytics_content_collections",
|
|
1056
|
+
{
|
|
1057
|
+
tenantKey: text("tenant_key").primaryKey(),
|
|
1058
|
+
ownerEmail: text("owner_email").notNull(),
|
|
1059
|
+
orgId: text("org_id"),
|
|
1060
|
+
enabled: boolean("enabled").notNull().default(false),
|
|
1061
|
+
config: text("config").notNull().default("{}"),
|
|
1062
|
+
createdAt: text("created_at").notNull().default(now()),
|
|
1063
|
+
updatedAt: text("updated_at").notNull().default(now()),
|
|
1064
|
+
},
|
|
1065
|
+
);
|
|
1066
|
+
|
|
1067
|
+
export const analyticsContentItems = table(
|
|
1068
|
+
"analytics_content_items",
|
|
1069
|
+
{
|
|
1070
|
+
id: text("id").primaryKey(),
|
|
1071
|
+
tenantKey: text("tenant_key").notNull(),
|
|
1072
|
+
ownerEmail: text("owner_email").notNull(),
|
|
1073
|
+
orgId: text("org_id"),
|
|
1074
|
+
source: text("source").notNull(),
|
|
1075
|
+
externalId: text("external_id").notNull(),
|
|
1076
|
+
url: text("url"),
|
|
1077
|
+
title: text("title"),
|
|
1078
|
+
author: text("author"),
|
|
1079
|
+
authors: text("authors").notNull().default("[]"),
|
|
1080
|
+
publishedAt: text("published_at"),
|
|
1081
|
+
relevance: text("relevance"),
|
|
1082
|
+
metadata: text("metadata").notNull().default("{}"),
|
|
1083
|
+
firstSeenAt: text("first_seen_at").notNull(),
|
|
1084
|
+
updatedAt: text("updated_at").notNull(),
|
|
1085
|
+
},
|
|
1086
|
+
(t) => ({
|
|
1087
|
+
sourceExternalUnique: uniqueIndex("analytics_content_items_key_idx").on(
|
|
1088
|
+
t.tenantKey,
|
|
1089
|
+
t.source,
|
|
1090
|
+
t.externalId,
|
|
1091
|
+
),
|
|
1092
|
+
tenantPublishedIdx: index("analytics_content_items_published_idx").on(
|
|
1093
|
+
t.tenantKey,
|
|
1094
|
+
t.publishedAt,
|
|
1095
|
+
),
|
|
1096
|
+
}),
|
|
1097
|
+
);
|
|
1098
|
+
|
|
1099
|
+
// A row exists only for a value a succeeded run collected. A missing row is
|
|
1100
|
+
// "not collected", never zero; the collection runs say which.
|
|
1101
|
+
export const analyticsContentMetricSnapshots = table(
|
|
1102
|
+
"analytics_content_metric_snapshots",
|
|
1103
|
+
{
|
|
1104
|
+
id: text("id").primaryKey(),
|
|
1105
|
+
tenantKey: text("tenant_key").notNull(),
|
|
1106
|
+
ownerEmail: text("owner_email").notNull(),
|
|
1107
|
+
orgId: text("org_id"),
|
|
1108
|
+
itemId: text("item_id").notNull(),
|
|
1109
|
+
metricDate: text("metric_date").notNull(),
|
|
1110
|
+
metric: text("metric").notNull(),
|
|
1111
|
+
value: real("value").notNull(),
|
|
1112
|
+
source: text("source").notNull(),
|
|
1113
|
+
runId: text("run_id").notNull(),
|
|
1114
|
+
collectedAt: text("collected_at").notNull(),
|
|
1115
|
+
},
|
|
1116
|
+
(t) => ({
|
|
1117
|
+
snapshotUnique: uniqueIndex(
|
|
1118
|
+
"analytics_content_metric_snapshots_key_idx",
|
|
1119
|
+
).on(t.tenantKey, t.itemId, t.metricDate, t.metric),
|
|
1120
|
+
tenantDateIdx: index("analytics_content_metric_snapshots_date_idx").on(
|
|
1121
|
+
t.tenantKey,
|
|
1122
|
+
t.metricDate,
|
|
1123
|
+
),
|
|
1124
|
+
}),
|
|
1125
|
+
);
|
|
1126
|
+
|
|
1127
|
+
export const analyticsContentCollectionRuns = table(
|
|
1128
|
+
"analytics_content_collection_runs",
|
|
1129
|
+
{
|
|
1130
|
+
id: text("id").primaryKey(),
|
|
1131
|
+
tenantKey: text("tenant_key").notNull(),
|
|
1132
|
+
ownerEmail: text("owner_email").notNull(),
|
|
1133
|
+
orgId: text("org_id"),
|
|
1134
|
+
step: text("step").notNull(),
|
|
1135
|
+
trigger: text("trigger").notNull(),
|
|
1136
|
+
status: text("status").notNull(),
|
|
1137
|
+
windowStart: text("window_start"),
|
|
1138
|
+
windowEnd: text("window_end"),
|
|
1139
|
+
startedAt: text("started_at").notNull(),
|
|
1140
|
+
finishedAt: text("finished_at"),
|
|
1141
|
+
rowCount: integer("row_count"),
|
|
1142
|
+
bytesProcessed: bigint("bytes_processed", { mode: "number" }),
|
|
1143
|
+
bytesCap: bigint("bytes_cap", { mode: "number" }),
|
|
1144
|
+
costUsd: real("cost_usd"),
|
|
1145
|
+
errorCode: text("error_code"),
|
|
1146
|
+
errorMessage: text("error_message"),
|
|
1147
|
+
},
|
|
1148
|
+
(t) => ({
|
|
1149
|
+
tenantStepStartedIdx: index(
|
|
1150
|
+
"analytics_content_collection_runs_step_idx",
|
|
1151
|
+
).on(t.tenantKey, t.step, t.startedAt),
|
|
1152
|
+
}),
|
|
1153
|
+
);
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import {
|
|
2
|
+
collectContentForAllTenants,
|
|
3
|
+
type ContentStepRunSummary,
|
|
4
|
+
} from "../lib/content-metrics/collect";
|
|
5
|
+
|
|
6
|
+
let running = false;
|
|
7
|
+
|
|
8
|
+
export type ContentCollectionPassResult =
|
|
9
|
+
| { ran: true; steps: ContentStepRunSummary[] }
|
|
10
|
+
| { ran: false; reason: "already_running" };
|
|
11
|
+
|
|
12
|
+
export async function runContentMetricsCollectionOnce(): Promise<ContentCollectionPassResult> {
|
|
13
|
+
if (running) return { ran: false, reason: "already_running" };
|
|
14
|
+
running = true;
|
|
15
|
+
try {
|
|
16
|
+
return { ran: true, steps: await collectContentForAllTenants() };
|
|
17
|
+
} finally {
|
|
18
|
+
running = false;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { runWithRequestContext } from "@agent-native/core/server";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
CONTENT_COLLECTION_STEPS,
|
|
5
|
+
stepEnabled,
|
|
6
|
+
type ContentCollectionStepId,
|
|
7
|
+
} from "./config.js";
|
|
8
|
+
import { contentRunFailureFromError, type ContentStep } from "./step.js";
|
|
9
|
+
import { builderBlogStep } from "./steps/builder-blog.js";
|
|
10
|
+
import { ga4Step } from "./steps/ga4.js";
|
|
11
|
+
import { searchConsoleStep } from "./steps/search-console.js";
|
|
12
|
+
import {
|
|
13
|
+
closeStaleContentRuns,
|
|
14
|
+
completeContentRun,
|
|
15
|
+
failContentRun,
|
|
16
|
+
listEnabledContentCollections,
|
|
17
|
+
startContentRun,
|
|
18
|
+
type ContentCollection,
|
|
19
|
+
type ContentRunStatus,
|
|
20
|
+
type ContentRunTrigger,
|
|
21
|
+
} from "./store.js";
|
|
22
|
+
|
|
23
|
+
const STEPS: Record<ContentCollectionStepId, ContentStep> = {
|
|
24
|
+
builder_blog: builderBlogStep,
|
|
25
|
+
ga4: ga4Step,
|
|
26
|
+
search_console: searchConsoleStep,
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export type ContentStepRunSummary = {
|
|
30
|
+
tenantKey: string;
|
|
31
|
+
step: ContentCollectionStepId;
|
|
32
|
+
status: ContentRunStatus | "skipped";
|
|
33
|
+
runId: string | null;
|
|
34
|
+
reason?: string;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export async function collectContentForCollection(
|
|
38
|
+
collection: ContentCollection,
|
|
39
|
+
options: {
|
|
40
|
+
trigger: ContentRunTrigger;
|
|
41
|
+
now?: Date;
|
|
42
|
+
signal?: AbortSignal;
|
|
43
|
+
steps?: ContentCollectionStepId[];
|
|
44
|
+
},
|
|
45
|
+
): Promise<ContentStepRunSummary[]> {
|
|
46
|
+
const now = options.now ?? new Date();
|
|
47
|
+
const summaries: ContentStepRunSummary[] = [];
|
|
48
|
+
const steps = options.steps ?? [...CONTENT_COLLECTION_STEPS];
|
|
49
|
+
|
|
50
|
+
for (const stepId of steps) {
|
|
51
|
+
if (!stepEnabled(collection.config, stepId)) {
|
|
52
|
+
summaries.push({
|
|
53
|
+
tenantKey: collection.tenantKey,
|
|
54
|
+
step: stepId,
|
|
55
|
+
status: "skipped",
|
|
56
|
+
runId: null,
|
|
57
|
+
reason: "step_disabled",
|
|
58
|
+
});
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const claim = await startContentRun(
|
|
62
|
+
collection,
|
|
63
|
+
stepId,
|
|
64
|
+
options.trigger,
|
|
65
|
+
now,
|
|
66
|
+
);
|
|
67
|
+
if ("reason" in claim) {
|
|
68
|
+
summaries.push({
|
|
69
|
+
tenantKey: collection.tenantKey,
|
|
70
|
+
step: stepId,
|
|
71
|
+
status: "skipped",
|
|
72
|
+
runId: null,
|
|
73
|
+
reason: claim.reason,
|
|
74
|
+
});
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const runId = claim.runId;
|
|
79
|
+
try {
|
|
80
|
+
const result = await runWithRequestContext(
|
|
81
|
+
{
|
|
82
|
+
userEmail: collection.ownerEmail,
|
|
83
|
+
...(collection.orgId ? { orgId: collection.orgId } : {}),
|
|
84
|
+
},
|
|
85
|
+
() =>
|
|
86
|
+
STEPS[stepId].run({
|
|
87
|
+
collection,
|
|
88
|
+
now,
|
|
89
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
90
|
+
}),
|
|
91
|
+
);
|
|
92
|
+
await completeContentRun(
|
|
93
|
+
collection,
|
|
94
|
+
runId,
|
|
95
|
+
stepId,
|
|
96
|
+
result,
|
|
97
|
+
result.outcome,
|
|
98
|
+
);
|
|
99
|
+
summaries.push({
|
|
100
|
+
tenantKey: collection.tenantKey,
|
|
101
|
+
step: stepId,
|
|
102
|
+
status: "succeeded",
|
|
103
|
+
runId,
|
|
104
|
+
});
|
|
105
|
+
} catch (error) {
|
|
106
|
+
const failure = contentRunFailureFromError(error);
|
|
107
|
+
await failContentRun(runId, failure);
|
|
108
|
+
summaries.push({
|
|
109
|
+
tenantKey: collection.tenantKey,
|
|
110
|
+
step: stepId,
|
|
111
|
+
status: failure.status,
|
|
112
|
+
runId,
|
|
113
|
+
reason: failure.errorCode,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return summaries;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** The daily pass over every tenant that turned content collection on. */
|
|
121
|
+
export async function collectContentForAllTenants(
|
|
122
|
+
options: { now?: Date; signal?: AbortSignal } = {},
|
|
123
|
+
): Promise<ContentStepRunSummary[]> {
|
|
124
|
+
await closeStaleContentRuns(options.now);
|
|
125
|
+
const summaries: ContentStepRunSummary[] = [];
|
|
126
|
+
let afterTenantKey: string | undefined;
|
|
127
|
+
for (;;) {
|
|
128
|
+
const collections = await listEnabledContentCollections(afterTenantKey);
|
|
129
|
+
for (const collection of collections) {
|
|
130
|
+
summaries.push(
|
|
131
|
+
...(await collectContentForCollection(collection, {
|
|
132
|
+
trigger: "scheduled",
|
|
133
|
+
...options,
|
|
134
|
+
})),
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
if (collections.length < 1000) break;
|
|
138
|
+
afterTenantKey = collections[collections.length - 1]!.tenantKey;
|
|
139
|
+
}
|
|
140
|
+
return summaries;
|
|
141
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Collection steps, in the order the daily job runs them. A step's id is the
|
|
5
|
+
* `step` stored on its collection runs and the `source` on what it writes.
|
|
6
|
+
*/
|
|
7
|
+
export const CONTENT_COLLECTION_STEPS = [
|
|
8
|
+
"builder_blog",
|
|
9
|
+
"ga4",
|
|
10
|
+
"search_console",
|
|
11
|
+
] as const;
|
|
12
|
+
export type ContentCollectionStepId = (typeof CONTENT_COLLECTION_STEPS)[number];
|
|
13
|
+
|
|
14
|
+
const stepToggleSchema = z.object({ enabled: z.boolean() });
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* A Search Console property: `sc-domain:example.com` or a URL-prefix
|
|
18
|
+
* property such as `https://www.example.com/`. `pathPrefix` keeps a large
|
|
19
|
+
* site to the pages the Content view tracks.
|
|
20
|
+
*/
|
|
21
|
+
export const searchConsoleSiteSchema = z.object({
|
|
22
|
+
siteUrl: z.string().trim().min(1),
|
|
23
|
+
pathPrefix: z.string().trim().startsWith("/").optional(),
|
|
24
|
+
});
|
|
25
|
+
export type SearchConsoleSite = z.infer<typeof searchConsoleSiteSchema>;
|
|
26
|
+
|
|
27
|
+
export const contentCollectionConfigSchema = z.object({
|
|
28
|
+
builderBlog: stepToggleSchema.default({ enabled: true }),
|
|
29
|
+
ga4: stepToggleSchema.default({ enabled: true }),
|
|
30
|
+
// Off until a property owner grants the service account access; a run
|
|
31
|
+
// before then would only record "no access".
|
|
32
|
+
searchConsole: stepToggleSchema
|
|
33
|
+
.extend({ sites: z.array(searchConsoleSiteSchema).max(20).default([]) })
|
|
34
|
+
.default({ enabled: false, sites: [] }),
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
export type ContentCollectionConfig = z.infer<
|
|
38
|
+
typeof contentCollectionConfigSchema
|
|
39
|
+
>;
|
|
40
|
+
|
|
41
|
+
export function defaultContentCollectionConfig(): ContentCollectionConfig {
|
|
42
|
+
return contentCollectionConfigSchema.parse({});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function stepEnabled(
|
|
46
|
+
config: ContentCollectionConfig,
|
|
47
|
+
step: ContentCollectionStepId,
|
|
48
|
+
): boolean {
|
|
49
|
+
switch (step) {
|
|
50
|
+
case "builder_blog":
|
|
51
|
+
return config.builderBlog.enabled;
|
|
52
|
+
case "ga4":
|
|
53
|
+
return config.ga4.enabled;
|
|
54
|
+
case "search_console":
|
|
55
|
+
return config.searchConsole.enabled;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export type ContentCollectionScope = {
|
|
60
|
+
ownerEmail: string;
|
|
61
|
+
orgId: string | null;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
/** Same tenant keys the first-party rollup tables use. */
|
|
65
|
+
export function contentTenantKey(scope: ContentCollectionScope): string {
|
|
66
|
+
return scope.orgId
|
|
67
|
+
? `org:${scope.orgId}`
|
|
68
|
+
: `user:${scope.ownerEmail.trim().toLowerCase()}`;
|
|
69
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// Created by a migration in server/plugins/db.ts. Hosted request paths never
|
|
2
|
+
// run DDL, so the statements live here and the migration list pastes them.
|
|
3
|
+
export const CONTENT_METRICS_MIGRATION_SQL = [
|
|
4
|
+
`CREATE TABLE IF NOT EXISTS analytics_content_collections (
|
|
5
|
+
tenant_key TEXT PRIMARY KEY,
|
|
6
|
+
owner_email TEXT NOT NULL,
|
|
7
|
+
org_id TEXT,
|
|
8
|
+
enabled BOOLEAN NOT NULL DEFAULT false,
|
|
9
|
+
config TEXT NOT NULL DEFAULT '{}',
|
|
10
|
+
created_at TEXT NOT NULL DEFAULT now(),
|
|
11
|
+
updated_at TEXT NOT NULL DEFAULT now()
|
|
12
|
+
)`,
|
|
13
|
+
`CREATE TABLE IF NOT EXISTS analytics_content_items (
|
|
14
|
+
id TEXT PRIMARY KEY,
|
|
15
|
+
tenant_key TEXT NOT NULL,
|
|
16
|
+
owner_email TEXT NOT NULL,
|
|
17
|
+
org_id TEXT,
|
|
18
|
+
source TEXT NOT NULL,
|
|
19
|
+
external_id TEXT NOT NULL,
|
|
20
|
+
url TEXT,
|
|
21
|
+
title TEXT,
|
|
22
|
+
author TEXT,
|
|
23
|
+
authors TEXT NOT NULL DEFAULT '[]',
|
|
24
|
+
published_at TEXT,
|
|
25
|
+
relevance TEXT,
|
|
26
|
+
metadata TEXT NOT NULL DEFAULT '{}',
|
|
27
|
+
first_seen_at TEXT NOT NULL,
|
|
28
|
+
updated_at TEXT NOT NULL
|
|
29
|
+
)`,
|
|
30
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS analytics_content_items_key_idx ON analytics_content_items (tenant_key, source, external_id)`,
|
|
31
|
+
`CREATE INDEX IF NOT EXISTS analytics_content_items_published_idx ON analytics_content_items (tenant_key, published_at)`,
|
|
32
|
+
`CREATE TABLE IF NOT EXISTS analytics_content_metric_snapshots (
|
|
33
|
+
id TEXT PRIMARY KEY,
|
|
34
|
+
tenant_key TEXT NOT NULL,
|
|
35
|
+
owner_email TEXT NOT NULL,
|
|
36
|
+
org_id TEXT,
|
|
37
|
+
item_id TEXT NOT NULL,
|
|
38
|
+
metric_date TEXT NOT NULL,
|
|
39
|
+
metric TEXT NOT NULL,
|
|
40
|
+
value DOUBLE PRECISION NOT NULL,
|
|
41
|
+
source TEXT NOT NULL,
|
|
42
|
+
run_id TEXT NOT NULL,
|
|
43
|
+
collected_at TEXT NOT NULL
|
|
44
|
+
)`,
|
|
45
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS analytics_content_metric_snapshots_key_idx ON analytics_content_metric_snapshots (tenant_key, item_id, metric_date, metric)`,
|
|
46
|
+
`CREATE INDEX IF NOT EXISTS analytics_content_metric_snapshots_date_idx ON analytics_content_metric_snapshots (tenant_key, metric_date)`,
|
|
47
|
+
`CREATE TABLE IF NOT EXISTS analytics_content_collection_runs (
|
|
48
|
+
id TEXT PRIMARY KEY,
|
|
49
|
+
tenant_key TEXT NOT NULL,
|
|
50
|
+
owner_email TEXT NOT NULL,
|
|
51
|
+
org_id TEXT,
|
|
52
|
+
step TEXT NOT NULL,
|
|
53
|
+
trigger TEXT NOT NULL,
|
|
54
|
+
status TEXT NOT NULL,
|
|
55
|
+
window_start TEXT,
|
|
56
|
+
window_end TEXT,
|
|
57
|
+
started_at TEXT NOT NULL,
|
|
58
|
+
finished_at TEXT,
|
|
59
|
+
row_count INTEGER,
|
|
60
|
+
bytes_processed BIGINT,
|
|
61
|
+
bytes_cap BIGINT,
|
|
62
|
+
cost_usd DOUBLE PRECISION,
|
|
63
|
+
error_code TEXT,
|
|
64
|
+
error_message TEXT
|
|
65
|
+
)`,
|
|
66
|
+
`CREATE INDEX IF NOT EXISTS analytics_content_collection_runs_step_idx ON analytics_content_collection_runs (tenant_key, step, started_at)`,
|
|
67
|
+
];
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import {
|
|
2
|
+
readServiceAccountIdentity,
|
|
3
|
+
type ServiceAccountIdentity,
|
|
4
|
+
} from "../google-analytics.js";
|
|
5
|
+
import {
|
|
6
|
+
CONTENT_COLLECTION_STEPS,
|
|
7
|
+
stepEnabled,
|
|
8
|
+
type ContentCollectionStepId,
|
|
9
|
+
} from "./config.js";
|
|
10
|
+
import {
|
|
11
|
+
latestContentRun,
|
|
12
|
+
readContentCollection,
|
|
13
|
+
type ContentCollection,
|
|
14
|
+
} from "./store.js";
|
|
15
|
+
|
|
16
|
+
export type ContentStepStatus = {
|
|
17
|
+
step: ContentCollectionStepId;
|
|
18
|
+
enabled: boolean;
|
|
19
|
+
lastRun: {
|
|
20
|
+
id: string;
|
|
21
|
+
trigger: string;
|
|
22
|
+
status: string;
|
|
23
|
+
startedAt: string;
|
|
24
|
+
finishedAt: string | null;
|
|
25
|
+
windowStart: string | null;
|
|
26
|
+
windowEnd: string | null;
|
|
27
|
+
rowCount: number | null;
|
|
28
|
+
bytesProcessed: number | null;
|
|
29
|
+
bytesCap: number | null;
|
|
30
|
+
errorCode: string | null;
|
|
31
|
+
errorMessage: string | null;
|
|
32
|
+
} | null;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
export type ContentCollectionStatus = {
|
|
36
|
+
configured: boolean;
|
|
37
|
+
enabled: boolean;
|
|
38
|
+
ownerEmail: string | null;
|
|
39
|
+
config: ContentCollection["config"] | null;
|
|
40
|
+
/** Who a Search Console or GA4 property owner grants access to. */
|
|
41
|
+
serviceAccount: ServiceAccountIdentity;
|
|
42
|
+
steps: ContentStepStatus[];
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export async function readContentCollectionStatus(
|
|
46
|
+
orgId: string,
|
|
47
|
+
userEmail: string,
|
|
48
|
+
): Promise<ContentCollectionStatus> {
|
|
49
|
+
const collection = await readContentCollection({
|
|
50
|
+
orgId,
|
|
51
|
+
ownerEmail: userEmail,
|
|
52
|
+
});
|
|
53
|
+
const serviceAccount = await readServiceAccountIdentity();
|
|
54
|
+
if (!collection) {
|
|
55
|
+
return {
|
|
56
|
+
configured: false,
|
|
57
|
+
enabled: false,
|
|
58
|
+
ownerEmail: null,
|
|
59
|
+
config: null,
|
|
60
|
+
serviceAccount,
|
|
61
|
+
steps: [],
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const steps: ContentStepStatus[] = [];
|
|
65
|
+
for (const step of CONTENT_COLLECTION_STEPS) {
|
|
66
|
+
const run = await latestContentRun(collection, step);
|
|
67
|
+
steps.push({
|
|
68
|
+
step,
|
|
69
|
+
enabled: stepEnabled(collection.config, step),
|
|
70
|
+
lastRun: run
|
|
71
|
+
? {
|
|
72
|
+
id: run.id,
|
|
73
|
+
trigger: run.trigger,
|
|
74
|
+
status: run.status,
|
|
75
|
+
startedAt: run.startedAt,
|
|
76
|
+
finishedAt: run.finishedAt,
|
|
77
|
+
windowStart: run.windowStart,
|
|
78
|
+
windowEnd: run.windowEnd,
|
|
79
|
+
rowCount: run.rowCount,
|
|
80
|
+
bytesProcessed: run.bytesProcessed,
|
|
81
|
+
bytesCap: run.bytesCap,
|
|
82
|
+
errorCode: run.errorCode,
|
|
83
|
+
errorMessage: run.errorMessage,
|
|
84
|
+
}
|
|
85
|
+
: null,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
configured: true,
|
|
90
|
+
enabled: collection.enabled,
|
|
91
|
+
ownerEmail: collection.ownerEmail,
|
|
92
|
+
config: collection.config,
|
|
93
|
+
serviceAccount,
|
|
94
|
+
steps,
|
|
95
|
+
};
|
|
96
|
+
}
|