@dragonmastery/tamer 0.1.1 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/README.md +569 -18
  2. package/dist/CFApiClient-DhbyyV71.mjs +868 -0
  3. package/dist/CFApiClient-DhbyyV71.mjs.map +1 -0
  4. package/dist/StateManager-DTqtLLVX.mjs +760 -0
  5. package/dist/StateManager-DTqtLLVX.mjs.map +1 -0
  6. package/dist/apply-B0b_jjGv.mjs +423 -0
  7. package/dist/apply-B0b_jjGv.mjs.map +1 -0
  8. package/dist/applyTarget-BetDYdeS.mjs +152 -0
  9. package/dist/applyTarget-BetDYdeS.mjs.map +1 -0
  10. package/dist/bootstrap-CBzPilB1.mjs +33 -0
  11. package/dist/bootstrap-CBzPilB1.mjs.map +1 -0
  12. package/dist/buildDispatchUploadForm-BoUB93b3.mjs +38 -0
  13. package/dist/buildDispatchUploadForm-BoUB93b3.mjs.map +1 -0
  14. package/dist/cloudflareSnapshot-B4FOaNr0.mjs +163 -0
  15. package/dist/cloudflareSnapshot-B4FOaNr0.mjs.map +1 -0
  16. package/dist/deploy-gHEQxhmx.mjs +119 -0
  17. package/dist/deploy-gHEQxhmx.mjs.map +1 -0
  18. package/dist/destroy-B21f3wgq.mjs +215 -0
  19. package/dist/destroy-B21f3wgq.mjs.map +1 -0
  20. package/dist/destroy-tenant-BW2nasnK.mjs +103 -0
  21. package/dist/destroy-tenant-BW2nasnK.mjs.map +1 -0
  22. package/dist/dev-Dt26nzMJ.mjs +103 -0
  23. package/dist/dev-Dt26nzMJ.mjs.map +1 -0
  24. package/dist/dns-records.resolve-C2T0m4NG.mjs +3 -0
  25. package/dist/dns-records.resolve-DwBR_1WI.mjs +47 -0
  26. package/dist/dns-records.resolve-DwBR_1WI.mjs.map +1 -0
  27. package/dist/dns-records.sync-Bpzz9H0s.mjs +75 -0
  28. package/dist/dns-records.sync-Bpzz9H0s.mjs.map +1 -0
  29. package/dist/doctor-C_hs7k2D.mjs +34 -0
  30. package/dist/doctor-C_hs7k2D.mjs.map +1 -0
  31. package/dist/drift-D5qzCTft.mjs +10 -0
  32. package/dist/drift-D8ZrSgTn.mjs +323 -0
  33. package/dist/drift-D8ZrSgTn.mjs.map +1 -0
  34. package/dist/events-BSwGdkGj.mjs +68 -0
  35. package/dist/events-BSwGdkGj.mjs.map +1 -0
  36. package/dist/fetchStackImports-B4ZJahOt.mjs +3817 -0
  37. package/dist/fetchStackImports-B4ZJahOt.mjs.map +1 -0
  38. package/dist/generator-CIMbcPzv.mjs +77 -0
  39. package/dist/generator-CIMbcPzv.mjs.map +1 -0
  40. package/dist/import-BrduwA9Z.mjs +164 -0
  41. package/dist/import-BrduwA9Z.mjs.map +1 -0
  42. package/dist/index.d.mts +6592 -56
  43. package/dist/index.d.mts.map +1 -1
  44. package/dist/index.mjs +18 -1
  45. package/dist/index.mjs.map +1 -0
  46. package/dist/loader-DP7yXqT6.mjs +518 -0
  47. package/dist/loader-DP7yXqT6.mjs.map +1 -0
  48. package/dist/logpush-job-xS7270FZ.mjs +1106 -0
  49. package/dist/logpush-job-xS7270FZ.mjs.map +1 -0
  50. package/dist/migrate-CahG6BYV.mjs +87 -0
  51. package/dist/migrate-CahG6BYV.mjs.map +1 -0
  52. package/dist/normalize-Bx0bpFop.mjs +236 -0
  53. package/dist/normalize-Bx0bpFop.mjs.map +1 -0
  54. package/dist/plan-DWvsvy1U.mjs +453 -0
  55. package/dist/plan-DWvsvy1U.mjs.map +1 -0
  56. package/dist/planFormat-CJw8Kq2s.mjs +119 -0
  57. package/dist/planFormat-CJw8Kq2s.mjs.map +1 -0
  58. package/dist/provision-tenant-WTKo93Y0.mjs +192 -0
  59. package/dist/provision-tenant-WTKo93Y0.mjs.map +1 -0
  60. package/dist/r2S3EmptyBucket-DD81ZWQ7.mjs +92 -0
  61. package/dist/r2S3EmptyBucket-DD81ZWQ7.mjs.map +1 -0
  62. package/dist/stackOutputs-W9mnnJuj.mjs +69 -0
  63. package/dist/stackOutputs-W9mnnJuj.mjs.map +1 -0
  64. package/dist/status-DLwREPjb.mjs +198 -0
  65. package/dist/status-DLwREPjb.mjs.map +1 -0
  66. package/dist/sync-f2K2blwm.mjs +90 -0
  67. package/dist/sync-f2K2blwm.mjs.map +1 -0
  68. package/dist/tamer.d.mts +1 -0
  69. package/dist/tamer.mjs +4553 -0
  70. package/dist/tamer.mjs.map +1 -0
  71. package/dist/tamerArtifactsR2-Ccgplu2Q.mjs +52 -0
  72. package/dist/tamerArtifactsR2-Ccgplu2Q.mjs.map +1 -0
  73. package/dist/types-CqxqYnrT.mjs +44 -0
  74. package/dist/types-CqxqYnrT.mjs.map +1 -0
  75. package/dist/verifyPlanFile-c16z1AMH.mjs +33 -0
  76. package/dist/verifyPlanFile-c16z1AMH.mjs.map +1 -0
  77. package/dist/wfp-delete-DysvX1u7.mjs +36 -0
  78. package/dist/wfp-delete-DysvX1u7.mjs.map +1 -0
  79. package/dist/wfp-put-jaVd_LjO.mjs +52 -0
  80. package/dist/wfp-put-jaVd_LjO.mjs.map +1 -0
  81. package/dist/worker-route-Be2IvOdr.mjs +263 -0
  82. package/dist/worker-route-Be2IvOdr.mjs.map +1 -0
  83. package/dist/workers-aGILs77X.mjs +87 -0
  84. package/dist/workers-aGILs77X.mjs.map +1 -0
  85. package/dist/wranglerSpawn-DmEz0ldT.mjs +24 -0
  86. package/dist/wranglerSpawn-DmEz0ldT.mjs.map +1 -0
  87. package/dist/zoneResolver-VoxLHM4N.mjs +32 -0
  88. package/dist/zoneResolver-VoxLHM4N.mjs.map +1 -0
  89. package/package.json +42 -4
@@ -0,0 +1,1106 @@
1
+ import { m as getLogpushJobs } from "./normalize-Bx0bpFop.mjs";
2
+ import { n as r2S3CredentialsFromEnv, t as emptyR2BucketViaS3 } from "./r2S3EmptyBucket-DD81ZWQ7.mjs";
3
+
4
+ //#region src/features/logpush-job/logpush-job.resolve.ts
5
+ const DEFAULT_WORKERS_TRACE_LOG_FIELD_NAMES = [
6
+ "Event",
7
+ "EventTimestampMs",
8
+ "Outcome",
9
+ "Exceptions",
10
+ "Logs",
11
+ "ScriptName"
12
+ ];
13
+ function logpushJobStateKey(tenantId, logicalName, env) {
14
+ return `logpush_job:t-${tenantId}-${logicalName}-${env}`;
15
+ }
16
+ function logpushJobCloudflareName(config, tenant, env) {
17
+ return config.jobName?.trim() || `tamer-${tenant.slug}-${config.logicalName}-${env}`;
18
+ }
19
+ function findR2BucketDerivedName(state, logicalName) {
20
+ for (const e of Object.values(state.getAll())) if (e.type === "r2_bucket" && e.logicalName === logicalName) return e.derivedName;
21
+ }
22
+ function buildR2WorkersTraceDestinationConf(args) {
23
+ const prefix = args.pathPrefix.replace(/^\/+|\/+$/g, "");
24
+ const path = prefix ? `${args.bucketDerivedName}/${prefix}/{DATE}` : `${args.bucketDerivedName}/{DATE}`;
25
+ const ak = encodeURIComponent(args.accessKeyId);
26
+ const sk = encodeURIComponent(args.secretAccessKey);
27
+ return `r2://${path}?account-id=${args.accountId}&access-key-id=${ak}&secret-access-key=${sk}`;
28
+ }
29
+ const PIPELINES_INGEST_HOST_SUFFIX = "ingest.cloudflare.com";
30
+ /**
31
+ * Pipelines list/create may return `endpoint` (full ingest URL). Prefer its
32
+ * origin for Logpush `destination_conf` so the hostname matches what Cloudflare
33
+ * registered for the stream.
34
+ */
35
+ function ingestOriginFromPipelinesStreamEndpoint(endpoint) {
36
+ const raw = endpoint?.trim();
37
+ if (!raw) return void 0;
38
+ try {
39
+ return new URL(raw).origin;
40
+ } catch {
41
+ return;
42
+ }
43
+ }
44
+ /** Normalize a Pipelines id to 32 lowercase hex digits (strip UUID dashes). */
45
+ function normalizePipelinesHexId(raw, label) {
46
+ const s = raw.trim().toLowerCase().replace(/-/g, "");
47
+ if (!/^[0-9a-f]{32}$/.test(s)) throw new Error(`${label} must be 32 hex characters (UUID with or without dashes); got "${raw}"`);
48
+ return s;
49
+ }
50
+ /**
51
+ * Logpush `destination_conf` for Workers trace → Pipelines stream HTTP ingest.
52
+ * Matches the shape produced by Cloudflare when creating a Pipelines Logpush job
53
+ * (`pipeline_id` + `header_Authorization` query params).
54
+ */
55
+ function buildPipelinesIngestDestinationConf(args) {
56
+ const pipeline = normalizePipelinesHexId(args.pipelineId, "pipelinesIngest.pipelineId");
57
+ const header = encodeURIComponent(`Bearer ${args.bearerToken}`);
58
+ return `${args.ingestBaseUrl?.trim().replace(/\/+$/, "") || `https://${normalizePipelinesHexId(args.streamId, "pipelinesIngest.streamId")}.${PIPELINES_INGEST_HOST_SUFFIX}`}?pipeline_id=${pipeline}&header_Authorization=${header}`;
59
+ }
60
+
61
+ //#endregion
62
+ //#region src/features/logpush-job/logpush-pipelines-key.ts
63
+ /** State row for a provisioned Pipelines graph backing a `pipelinesAuto` Logpush job. */
64
+ function logpushPipelinesStateKey(tenantId, logicalName, env) {
65
+ return `logpush_pipelines:t-${tenantId}-${logicalName}-${env}`;
66
+ }
67
+
68
+ //#endregion
69
+ //#region src/features/logpush-job/logpush-pipelines-trace-schema.ts
70
+ /**
71
+ * Pipelines stream schema for `workers_trace_events` (matches Logpush field set).
72
+ * @see https://developers.cloudflare.com/logs/logpush/logpush-job/datasets/workers-trace/
73
+ */
74
+ const WORKERS_TRACE_PIPELINES_SCHEMA_FIELDS = [
75
+ {
76
+ name: "CPUTimeMs",
77
+ type: "int64",
78
+ required: false
79
+ },
80
+ {
81
+ name: "DispatchNamespace",
82
+ type: "string",
83
+ required: false
84
+ },
85
+ {
86
+ name: "Entrypoint",
87
+ type: "string",
88
+ required: false
89
+ },
90
+ {
91
+ name: "Event",
92
+ type: "json",
93
+ required: false
94
+ },
95
+ {
96
+ name: "EventTimestampMs",
97
+ type: "int64",
98
+ required: false
99
+ },
100
+ {
101
+ name: "EventType",
102
+ type: "string",
103
+ required: false
104
+ },
105
+ {
106
+ name: "Exceptions",
107
+ type: "list",
108
+ required: false,
109
+ items: {
110
+ name: "item",
111
+ type: "json"
112
+ }
113
+ },
114
+ {
115
+ name: "Logs",
116
+ type: "list",
117
+ required: false,
118
+ items: {
119
+ name: "item",
120
+ type: "json"
121
+ }
122
+ },
123
+ {
124
+ name: "Outcome",
125
+ type: "string",
126
+ required: false
127
+ },
128
+ {
129
+ name: "ScriptName",
130
+ type: "string",
131
+ required: false
132
+ },
133
+ {
134
+ name: "ScriptTags",
135
+ type: "list",
136
+ required: false,
137
+ items: {
138
+ name: "item",
139
+ type: "string"
140
+ }
141
+ },
142
+ {
143
+ name: "ScriptVersion",
144
+ type: "json",
145
+ required: false
146
+ },
147
+ {
148
+ name: "WallTimeMs",
149
+ type: "int64",
150
+ required: false
151
+ }
152
+ ];
153
+ function buildWorkersTracePipelinesStreamBody(streamName) {
154
+ return {
155
+ name: streamName,
156
+ format: {
157
+ type: "json",
158
+ timestamp_format: "rfc3339"
159
+ },
160
+ schema: { fields: WORKERS_TRACE_PIPELINES_SCHEMA_FIELDS },
161
+ http: {
162
+ enabled: true,
163
+ authentication: true,
164
+ cors: {}
165
+ },
166
+ worker_binding: { enabled: true }
167
+ };
168
+ }
169
+
170
+ //#endregion
171
+ //#region src/features/logpush-job/logpush-pipelines-account-tokens.ts
172
+ const MAX_TOKEN_NAME = 200;
173
+ function trunc$1(s, max) {
174
+ return s.length <= max ? s : s.slice(0, max);
175
+ }
176
+ function findGroupId(groups, match, label) {
177
+ const g = groups.find((x) => match(x.name));
178
+ if (g) return g.id;
179
+ const sample = groups.filter((x) => /r2|pipeline|data catalog|send/i.test(x.name)).map((x) => x.name).slice(0, 12);
180
+ throw new Error(`logpushJobs pipelinesAuto: could not find permission group for ${label} (R2 / Pipelines names change over time). Sample matching names: ${sample.length ? sample.join("; ") : "(none)"}. Total groups listed: ${groups.length}.`);
181
+ }
182
+ async function resolvePipelinesAutoPermissionGroups(api) {
183
+ const groups = await api.accountTokenPermissionGroupsListAll();
184
+ return {
185
+ r2DataCatalogId: findGroupId(groups, (n) => n === "Workers R2 Data Catalog Write" || n === "Workers R2 Data Catalog Edit" || /r2/i.test(n) && /data catalog/i.test(n) && /(write|edit)/i.test(n), "Workers R2 Data Catalog (Edit / Write — same as dashboard token builder)"),
186
+ r2AccountStorageWriteId: findGroupId(groups, (n) => !/bucket item/i.test(n) && (n === "Workers R2 Storage Write" || n === "Workers R2 Storage Edit" || /workers r2 storage/i.test(n) && /(write|edit)/i.test(n)), "Workers R2 Storage Write (account — required for R2 Data Catalog warehouse / Iceberg data paths)"),
187
+ pipelinesSendId: findGroupId(groups, (n) => /pipelines/i.test(n) && /send/i.test(n) || n === "Workers Pipelines Send", "Workers Pipelines Send (stream ingest)")
188
+ };
189
+ }
190
+ /**
191
+ * Create two account tokens to match the Cloudflare dashboard:
192
+ * - **Pipelines Send** only (Logpush → stream HTTP ingest).
193
+ * - **R2 Data Catalog** only (account scope), for `POST …/r2-catalog/…/credential` and
194
+ * the `r2_data_catalog` sink `config.token` — like `[Pipelines] Data Catalog: …` in the UI.
195
+ */
196
+ async function mintPipelinesAutoAccountTokens(api, accountId, tokenNamePrefix) {
197
+ console.log("pipelinesAuto: fetching API token permission groups (paginated) and minting two account tokens…");
198
+ const g = await resolvePipelinesAutoPermissionGroups(api);
199
+ const accountRes = `com.cloudflare.api.account.${accountId}`;
200
+ const dataCatalogName = trunc$1(`${tokenNamePrefix} data-catalog`, MAX_TOKEN_NAME);
201
+ const r2Result = await api.accountTokenCreate({
202
+ name: dataCatalogName,
203
+ policies: [{
204
+ effect: "allow",
205
+ permission_groups: [{ id: g.r2DataCatalogId }, { id: g.r2AccountStorageWriteId }],
206
+ resources: { [accountRes]: "*" }
207
+ }]
208
+ });
209
+ if (!r2Result.value) throw new Error("Account Token API: expected result.value (secret) for R2 Data Catalog token — token create response incomplete");
210
+ const sendName = trunc$1(`${tokenNamePrefix} pipelines-send`, MAX_TOKEN_NAME);
211
+ const sendResult = await api.accountTokenCreate({
212
+ name: sendName,
213
+ policies: [{
214
+ effect: "allow",
215
+ permission_groups: [{ id: g.pipelinesSendId }],
216
+ resources: { [accountRes]: "*" }
217
+ }]
218
+ });
219
+ if (!sendResult.value) {
220
+ try {
221
+ await api.accountTokenDelete(r2Result.id);
222
+ } catch {}
223
+ throw new Error("Account Token API: expected result.value (secret) for Pipelines Send token — token create response incomplete");
224
+ }
225
+ return {
226
+ r2CatalogTokenId: r2Result.id,
227
+ r2CatalogTokenValue: r2Result.value,
228
+ pipelinesSendTokenId: sendResult.id,
229
+ pipelinesSendTokenValue: sendResult.value
230
+ };
231
+ }
232
+ /**
233
+ * Display names (after truncation) for the two minted account tokens. Must
234
+ * match {@link mintPipelinesAutoAccountTokens} and `tokenPrefix` in
235
+ * `ensurePipelinesLogpushProvision` (`Tamer {slug} {logical} {env}`).
236
+ */
237
+ function derivePipelinesAutoMintedTokenNames(tenant, logicalName, env) {
238
+ const tokenPrefix = `Tamer ${tenant.slug} ${logicalName} ${env}`;
239
+ return {
240
+ dataCatalog: trunc$1(`${tokenPrefix} data-catalog`, MAX_TOKEN_NAME),
241
+ pipelinesSend: trunc$1(`${tokenPrefix} pipelines-send`, MAX_TOKEN_NAME)
242
+ };
243
+ }
244
+ /**
245
+ * List account API tokens and revoke any that match the minted `pipelinesAuto`
246
+ * names. Use on destroy when state did not have token ids, deletes failed, or
247
+ * ids were wrong — same names as at mint time.
248
+ */
249
+ async function reconcilePipelinesAutoMintedAccountTokensByName(api, tenant, logicalName, env) {
250
+ const { dataCatalog, pipelinesSend } = derivePipelinesAutoMintedTokenNames(tenant, logicalName, env);
251
+ const legacyDataCatalog = trunc$1(`${`Tamer ${tenant.slug} ${logicalName} ${env}`} r2+catalog+sink`, MAX_TOKEN_NAME);
252
+ const want = new Set([
253
+ dataCatalog,
254
+ pipelinesSend,
255
+ legacyDataCatalog
256
+ ]);
257
+ const tokens = await api.accountTokenListAll();
258
+ for (const t of tokens) {
259
+ const name = t.name ?? "";
260
+ if (!t.id || !want.has(name)) continue;
261
+ try {
262
+ await api.accountTokenDelete(t.id);
263
+ console.log(`Pipelines: destroy reconcile — revoked account API token "${name}" [id ${t.id}].`);
264
+ } catch (e) {
265
+ console.warn(`Pipelines: destroy reconcile — account API token "${name}" [id ${t.id}]:`, e instanceof Error ? e.message : e);
266
+ }
267
+ }
268
+ }
269
+
270
+ //#endregion
271
+ //#region src/features/logpush-job/pipelinesSinkCatalogTableError.ts
272
+ /**
273
+ * Pipelines v1 `r2_data_catalog` sink HTTP 422 / code 1012: existing Iceberg
274
+ * table in the R2 Data Catalog — not “wrong token”.
275
+ */
276
+ function isPipelinesSinkExistingCatalogTableError(err) {
277
+ const m = err instanceof Error ? err.message : String(err);
278
+ return /\b(code:\s*1012|1012)\b/i.test(m) && /existing catalog|not yet supported|sink definition was invalid/i.test(m);
279
+ }
280
+ /**
281
+ * Actionable copy for `tamer apply` when Cloudflare refuses to create a sink
282
+ * that would write to an existing table.
283
+ */
284
+ function pipelinesSink1012RecoveryMessage(logicalName, namespace, tableName, catalogBucket) {
285
+ return [
286
+ `logpushJobs.${logicalName}: Pipelines r2_data_catalog sink was rejected (HTTP 422, code 1012).`,
287
+ `The Iceberg table "${namespace}" / "${tableName}" already exists in the R2 Data Catalog for bucket "${catalogBucket}".`,
288
+ `This API path only creates a new table — it does not attach to an existing one.`,
289
+ ``,
290
+ `What to do:`,
291
+ ` • In tamer/project.config.ts, set a new pipelinesAuto.tableName (e.g. "${tableName}_v2" or a dated suffix), and/or change pipelinesAuto.namespace.`,
292
+ ` • Or use a new catalog R2 bucket with no prior Iceberg layout.`,
293
+ ` • Removing old catalog data in the bucket is possible but advanced — only if you intend to delete that data.`
294
+ ].join("\n");
295
+ }
296
+
297
+ //#endregion
298
+ //#region src/features/logpush-job/logpush-pipelines-provision.ts
299
+ const MAX_CF_NAME = 200;
300
+ /**
301
+ * Resolves the Iceberg `table_name` for a **new** `r2_data_catalog` sink.
302
+ * When `tableNameAppendTimestamp` is not `false`, appends `_${ms}` to avoid
303
+ * HTTP 422 / 1012 if a table with the same base name is still in the catalog.
304
+ */
305
+ function pipelinesAutoIcebergTableNameForNewSink(baseTable, tableNameAppendTimestamp, now = Date.now()) {
306
+ const base = baseTable.trim();
307
+ if (!base) throw new Error("pipelinesAuto: tableName (base) is required");
308
+ if (tableNameAppendTimestamp === false) return base;
309
+ return `${base}_${now}`;
310
+ }
311
+ /**
312
+ * Pipelines `GET /sinks/{id}` is the source of truth for how the catalog
313
+ * registered the table (may differ from the `table_name` sent on create, e.g.
314
+ * suffix normalization).
315
+ */
316
+ function r2DataCatalogFromSinkGet(result) {
317
+ if (result.type !== "r2_data_catalog" || !result.config) return {};
318
+ const c = result.config;
319
+ const tableName = typeof c.table_name === "string" ? c.table_name.trim() : void 0;
320
+ const namespace = typeof c.namespace === "string" ? c.namespace.trim() : void 0;
321
+ return {
322
+ tableName: tableName || void 0,
323
+ namespace: namespace || void 0
324
+ };
325
+ }
326
+ /**
327
+ * Pipelines may report `worker_trace_events_<ms>` from create/GET while R2 SQL
328
+ * `SHOW TABLES` only lists the base name (e.g. `worker_trace_events`). Use the
329
+ * base + long-numeric-suffix pattern so stack outputs match R2 SQL.
330
+ */
331
+ function normalizeIcebergTableNameForR2SqlState(pipelinesReported, configBaseTable) {
332
+ const p = pipelinesReported.trim();
333
+ const base = configBaseTable.trim();
334
+ if (!base) return p;
335
+ if (p === base) return p;
336
+ const escaped = base.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
337
+ if ((/* @__PURE__ */ new RegExp(`^${escaped}_(\\d{10,})$`)).test(p)) return base;
338
+ return p;
339
+ }
340
+ /**
341
+ * R2 Data Catalog `GET …/r2-catalog` and `POST …/r2-catalog/…/enable` use
342
+ * **CLOUDFLARE_API_TOKEN** (apply token) — not the minted sub-tokens.
343
+ */
344
+ function throwPipelinesAutoR2CatalogApplyTokenError(logicalName, step, cause) {
345
+ const orig = cause instanceof Error ? cause.message : String(cause);
346
+ throw new Error(`logpushJobs.${logicalName} pipelinesAuto: ${step} — your **apply** API token (CLOUDFLARE_API_TOKEN) must be allowed to use the **R2 Data Catalog** account API (\`/accounts/.../r2-catalog/...\` — e.g. **Workers R2 Data Catalog: Edit** in the API token permissions for this account). That is separate from **Account API Tokens: Edit** (used only to mint the two sub-tokens) and separate from those minted tokens, which are only for the catalog credential + Pipelines. Original: ${orig}`);
347
+ }
348
+ function safeIdPart(s) {
349
+ return s.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
350
+ }
351
+ function trunc(s) {
352
+ return s.length <= MAX_CF_NAME ? s : s.slice(0, MAX_CF_NAME);
353
+ }
354
+ /** Stream / sink / pipeline names safe for Pipelines SQL identifiers. */
355
+ function derivePipelinesLogpushResourceNames(tenant, logicalName, env) {
356
+ const base = trunc(`tamer_${safeIdPart(tenant.slug)}_${safeIdPart(logicalName)}_${safeIdPart(env)}_wtr`);
357
+ return {
358
+ streamName: trunc(`${base}_stream`),
359
+ sinkName: trunc(`${base}_sink`),
360
+ pipelineName: trunc(`${base}_pl`)
361
+ };
362
+ }
363
+ /**
364
+ * Idempotent: enables R2 catalog, stores credentials, creates stream, sink, pipeline.
365
+ */
366
+ async function ensurePipelinesLogpushProvision(config, auto, tenant, env, accountId, state, api) {
367
+ const pkey = logpushPipelinesStateKey(tenant.id, config.logicalName, env);
368
+ const prior = state.get(pkey);
369
+ const tokenPrefix = `Tamer ${tenant.slug} ${config.logicalName} ${env}`;
370
+ const bucketDerived = findR2BucketDerivedName(state, auto.catalogBucketLogicalName);
371
+ if (!bucketDerived) throw new Error(`logpushJobs.${config.logicalName}: R2 bucket "${auto.catalogBucketLogicalName}" is not in state — apply R2 first`);
372
+ let r2CatId = prior?.type === "logpush_pipelines" ? prior.mintedR2CatalogTokenId : void 0;
373
+ let r2CatVal = prior?.type === "logpush_pipelines" ? prior.mintedR2CatalogTokenValue : void 0;
374
+ let sendId = prior?.type === "logpush_pipelines" ? prior.mintedPipelinesSendTokenId : void 0;
375
+ let sendVal = prior?.type === "logpush_pipelines" ? prior.mintedPipelinesSendTokenValue : void 0;
376
+ if (!r2CatId || !r2CatVal || !sendId || !sendVal) {
377
+ if (prior?.type === "logpush_pipelines" && (r2CatId || r2CatVal || sendId || sendVal)) console.warn(`logpushJobs.${config.logicalName}: state has partial minted token fields — creating fresh account tokens; old token ids may need manual cleanup in the dashboard.`);
378
+ const minted = await mintPipelinesAutoAccountTokens(api, accountId, tokenPrefix);
379
+ r2CatId = minted.r2CatalogTokenId;
380
+ r2CatVal = minted.r2CatalogTokenValue;
381
+ sendId = minted.pipelinesSendTokenId;
382
+ sendVal = minted.pipelinesSendTokenValue;
383
+ }
384
+ const cred = r2CatVal;
385
+ console.log(`logpushJobs.${config.logicalName}: enabling R2 Data Catalog on \"${bucketDerived}\" (uses **CLOUDFLARE_API_TOKEN**; see README)…`);
386
+ try {
387
+ await api.r2DataCatalogEnable(bucketDerived);
388
+ } catch (e) {
389
+ const msg = e instanceof Error ? e.message : String(e);
390
+ if (!/exists|already|active/i.test(msg)) throwPipelinesAutoR2CatalogApplyTokenError(config.logicalName, "R2 Data Catalog enable (POST .../r2-catalog/{bucket}/enable)", e);
391
+ }
392
+ try {
393
+ await api.r2DataCatalogStoreCredential(bucketDerived, cred);
394
+ } catch (e) {
395
+ const msg = e instanceof Error ? e.message : String(e);
396
+ if (!/exists|already|present/i.test(msg)) throwPipelinesAutoR2CatalogApplyTokenError(config.logicalName, "store R2 Data Catalog credentials (POST .../r2-catalog/{bucket}/credential)", e);
397
+ }
398
+ const ns = (auto.namespace ?? "default").trim() || "default";
399
+ const baseTable = auto.tableName.trim();
400
+ if (!baseTable) throw new Error(`logpushJobs.${config.logicalName}: pipelinesAuto.tableName is required`);
401
+ const names = derivePipelinesLogpushResourceNames(tenant, config.logicalName, env);
402
+ let stream = (await api.pipelinesV1StreamListAll()).find((s) => s.name === names.streamName);
403
+ let streamCreatedFresh = false;
404
+ if (!stream) {
405
+ const body = buildWorkersTracePipelinesStreamBody(names.streamName);
406
+ stream = await api.pipelinesV1StreamCreate(body);
407
+ streamCreatedFresh = true;
408
+ console.log(`Created Pipelines stream "${names.streamName}" [id ${stream.id}] (Workers trace schema).`);
409
+ } else console.log(`Pipelines stream "${names.streamName}" already exists [id ${stream.id}].`);
410
+ const streamIngestBaseUrl = ingestOriginFromPipelinesStreamEndpoint(stream.endpoint);
411
+ /** Logpush validates HTTPS to ingest; new stream subdomains can lag DNS briefly. */
412
+ if (streamCreatedFresh) {
413
+ const settleMs = 2500;
414
+ console.log(`logpushJobs.${config.logicalName}: waiting ${settleMs}ms for new stream ingest hostname before Logpush job step…`);
415
+ await new Promise((r) => setTimeout(r, settleMs));
416
+ }
417
+ const sinks = await api.pipelinesV1SinkListAll();
418
+ const rowGroup = auto.sinkRowGroupBytes ?? 134217728;
419
+ const fileSize = auto.sinkRollingFileSizeBytes ?? 100 * 1024 * 1024;
420
+ const interval = auto.sinkRollingIntervalSeconds ?? 300;
421
+ let sink = sinks.find((s) => s.name === names.sinkName);
422
+ let r2DataCatalogTableName = prior?.type === "logpush_pipelines" ? prior.r2DataCatalogTableName : void 0;
423
+ let r2DataCatalogTableNamePipelines = prior?.type === "logpush_pipelines" ? prior.r2DataCatalogTableNamePipelines : void 0;
424
+ if (!sink) {
425
+ const icebergTable = pipelinesAutoIcebergTableNameForNewSink(baseTable, auto.tableNameAppendTimestamp);
426
+ try {
427
+ sink = await api.pipelinesV1SinkCreate({
428
+ name: names.sinkName,
429
+ type: "r2_data_catalog",
430
+ format: {
431
+ type: "parquet",
432
+ compression: "zstd",
433
+ row_group_bytes: rowGroup
434
+ },
435
+ schema: { fields: [] },
436
+ config: {
437
+ token: cred,
438
+ account_id: accountId,
439
+ bucket: bucketDerived,
440
+ namespace: ns,
441
+ table_name: icebergTable,
442
+ rolling_policy: {
443
+ file_size_bytes: fileSize,
444
+ interval_seconds: interval
445
+ }
446
+ }
447
+ });
448
+ } catch (e) {
449
+ if (isPipelinesSinkExistingCatalogTableError(e)) throw new Error(pipelinesSink1012RecoveryMessage(config.logicalName, ns, icebergTable, bucketDerived), { cause: e });
450
+ throw e;
451
+ }
452
+ r2DataCatalogTableName = icebergTable;
453
+ console.log(`Created Pipelines sink "${names.sinkName}" (r2_data_catalog) [id ${sink.id}] → Iceberg table "${ns}" / "${icebergTable}".`);
454
+ } else console.log(`Pipelines sink "${names.sinkName}" already exists [id ${sink.id}].`);
455
+ let catalogNs = ns;
456
+ try {
457
+ const fromApi = r2DataCatalogFromSinkGet(await api.pipelinesV1SinkGet(sink.id));
458
+ if (fromApi.tableName) r2DataCatalogTableName = fromApi.tableName;
459
+ if (fromApi.namespace) catalogNs = fromApi.namespace;
460
+ } catch (e) {
461
+ const msg = e instanceof Error ? e.message : String(e);
462
+ console.warn(`logpushJobs.${config.logicalName}: Pipelines GET sink not used (${msg}) — Iceberg name falls back to create-time value in state; re-run apply after fixing API access if needed.`);
463
+ }
464
+ if (r2DataCatalogTableName) {
465
+ const raw = r2DataCatalogTableName;
466
+ r2DataCatalogTableNamePipelines = raw;
467
+ const normalized = normalizeIcebergTableNameForR2SqlState(raw, baseTable);
468
+ if (normalized !== raw) console.log(`logpushJobs.${config.logicalName}: R2 SQL table name for stack outputs: "${raw}" → "${normalized}" (Pipelines reports a suffixed id; catalog uses the base name).`);
469
+ r2DataCatalogTableName = normalized;
470
+ }
471
+ const sql = `INSERT INTO ${names.sinkName} SELECT * FROM ${names.streamName};`;
472
+ let pipeline = (await api.pipelineListAll()).find((p) => p.name === names.pipelineName);
473
+ if (!pipeline) {
474
+ pipeline = await api.pipelineCreate({
475
+ name: names.pipelineName,
476
+ sql
477
+ });
478
+ console.log(`Created Pipelines job "${names.pipelineName}" [id ${pipeline.id}].`);
479
+ } else if (pipeline.sql?.trim() !== sql.trim()) console.warn(`Pipelines job "${names.pipelineName}" exists with different SQL — not updating (replace manually or delete & re-apply).`);
480
+ else console.log(`Pipelines job "${names.pipelineName}" already exists [id ${pipeline.id}].`);
481
+ const ts = (/* @__PURE__ */ new Date()).toISOString();
482
+ const existing = state.get(pkey);
483
+ const entry = {
484
+ type: "logpush_pipelines",
485
+ logicalName: config.logicalName,
486
+ streamId: stream.id,
487
+ streamIngestBaseUrl,
488
+ sinkId: sink.id,
489
+ pipelineId: pipeline.id,
490
+ streamName: names.streamName,
491
+ sinkName: names.sinkName,
492
+ pipelineName: names.pipelineName,
493
+ r2DataCatalogTableName,
494
+ r2DataCatalogTableNamePipelines,
495
+ r2DataCatalogNamespace: catalogNs,
496
+ catalogBucketDerivedName: bucketDerived,
497
+ mintedR2CatalogTokenId: r2CatId,
498
+ mintedR2CatalogTokenValue: r2CatVal,
499
+ mintedPipelinesSendTokenId: sendId,
500
+ mintedPipelinesSendTokenValue: sendVal,
501
+ createdAt: existing?.type === "logpush_pipelines" ? existing.createdAt : ts,
502
+ updatedAt: ts
503
+ };
504
+ state.set(pkey, entry);
505
+ }
506
+ function samePipelinesV1Id(a, b) {
507
+ return a.replace(/-/g, "").toLowerCase() === b.replace(/-/g, "").toLowerCase();
508
+ }
509
+ /**
510
+ * Pipelines v1 may 404 on DELETE when state stores 32-hex ids but the path must
511
+ * be a hyphenated UUID, or the id drifts — list by name and retry once.
512
+ */
513
+ async function deletePipelinesV1StreamWithLookup(api, streamId, streamName) {
514
+ try {
515
+ await api.pipelinesV1StreamDelete(streamId);
516
+ return;
517
+ } catch (e) {
518
+ const streams = await api.pipelinesV1StreamListAll();
519
+ const hit = streams.find((s) => s.name === streamName) ?? streams.find((s) => samePipelinesV1Id(s.id, streamId));
520
+ if (hit) {
521
+ await api.pipelinesV1StreamDelete(hit.id);
522
+ return;
523
+ }
524
+ throw e;
525
+ }
526
+ }
527
+ async function deletePipelinesV1SinkWithLookup(api, sinkId, sinkName) {
528
+ try {
529
+ await api.pipelinesV1SinkDelete(sinkId);
530
+ return;
531
+ } catch (e) {
532
+ const sinks = await api.pipelinesV1SinkListAll();
533
+ const hit = sinks.find((s) => s.name === sinkName) ?? sinks.find((s) => samePipelinesV1Id(s.id, sinkId));
534
+ if (hit) {
535
+ await api.pipelinesV1SinkDelete(hit.id);
536
+ return;
537
+ }
538
+ throw e;
539
+ }
540
+ }
541
+ /**
542
+ * Best-effort teardown: SQL pipeline, then sink, then stream (after Logpush job is gone).
543
+ * Order avoids leaving the sink (e.g. r2_data_catalog) while the stream still exists.
544
+ */
545
+ async function deletePipelinesLogpushGraph(api, entry) {
546
+ const { pipelineId, streamId, sinkId } = entry;
547
+ for (const [label, id] of [["Pipelines Send", entry.mintedPipelinesSendTokenId], ["Data Catalog (minted)", entry.mintedR2CatalogTokenId]]) {
548
+ if (!id) continue;
549
+ try {
550
+ await api.accountTokenDelete(id);
551
+ console.log(`Deleted account API token [${label}] [id ${id}].`);
552
+ } catch (e) {
553
+ console.warn(`Account API token delete [${label}] [id ${id}]:`, e instanceof Error ? e.message : e);
554
+ }
555
+ }
556
+ try {
557
+ await api.pipelineDelete(pipelineId);
558
+ console.log(`Deleted Pipelines job "${entry.pipelineName}" [id ${pipelineId}].`);
559
+ } catch (e) {
560
+ console.warn(`Pipelines job delete [id ${pipelineId}]:`, e instanceof Error ? e.message : e);
561
+ }
562
+ try {
563
+ await deletePipelinesV1SinkWithLookup(api, sinkId, entry.sinkName);
564
+ console.log(`Deleted Pipelines sink "${entry.sinkName}" [id ${sinkId}].`);
565
+ } catch (e) {
566
+ console.warn(`Pipelines sink delete [id ${sinkId}]:`, e instanceof Error ? e.message : e);
567
+ }
568
+ try {
569
+ await deletePipelinesV1StreamWithLookup(api, streamId, entry.streamName);
570
+ console.log(`Deleted Pipelines stream "${entry.streamName}" [id ${streamId}].`);
571
+ } catch (e) {
572
+ console.warn(`Pipelines stream delete [id ${streamId}]:`, e instanceof Error ? e.message : e);
573
+ }
574
+ await pipelinesAutoTeardownR2DataCatalog(api, entry.catalogBucketDerivedName);
575
+ }
576
+ /**
577
+ * Stopping the Pipelines `r2_data_catalog` sink does not drop the Iceberg
578
+ * table. Disable the Data Catalog, then (when `R2_ACCESS_KEY_ID` is set) empty
579
+ * the catalog bucket so a later `tamer apply` is not blocked by HTTP 422 / 1012
580
+ * (table already exists in this catalog).
581
+ */
582
+ async function pipelinesAutoTeardownR2DataCatalog(api, catalogBucketDerivedName) {
583
+ try {
584
+ await api.r2DataCatalogDisable(catalogBucketDerivedName);
585
+ console.log(`Pipelines: disabled R2 Data Catalog on "${catalogBucketDerivedName}" (pipelinesAuto teardown).`);
586
+ } catch (e) {
587
+ const msg = e instanceof Error ? e.message : String(e);
588
+ if (!/\b(404|10006|not found)\b/i.test(msg)) console.warn(`Pipelines: R2 Data Catalog disable for "${catalogBucketDerivedName}": ${msg}`);
589
+ }
590
+ const s3creds = r2S3CredentialsFromEnv();
591
+ if (!s3creds) {
592
+ console.warn("Pipelines: set R2_ACCESS_KEY_ID and R2_SECRET_ACCESS_KEY to empty the catalog bucket and remove Iceberg table metadata; otherwise a future `tamer apply` may hit HTTP 422 / 1012 (table already exists). Alternatively bump `pipelinesAuto.tableName` or use a new catalog bucket.");
593
+ return;
594
+ }
595
+ try {
596
+ const accountId = api.getAccountId();
597
+ console.log(`Pipelines: emptying catalog bucket "${catalogBucketDerivedName}" via S3 (removes Iceberg layout)…`);
598
+ const { objectsDeleted, uploadsAborted } = await emptyR2BucketViaS3(accountId, catalogBucketDerivedName, s3creds);
599
+ console.log(`Pipelines: catalog bucket empty — ${objectsDeleted} object(s), ${uploadsAborted} incomplete multipart upload(s) aborted.`);
600
+ } catch (e) {
601
+ console.warn(`Pipelines: S3 empty of catalog bucket "${catalogBucketDerivedName}":`, e instanceof Error ? e.message : e);
602
+ }
603
+ }
604
+ /**
605
+ * After state-driven `deletePipelinesLogpushGraph`, or when `logpush_pipelines`
606
+ * state is missing, delete the SQL pipeline / sink / stream by the names derived
607
+ * from `derivePipelinesLogpushResourceNames` (config + env). Uses live ids from
608
+ * list APIs so stale state ids do not block teardown.
609
+ */
610
+ async function reconcilePipelinesAutoCloudResourcesByName(api, tenant, logicalName, env, opts) {
611
+ const names = derivePipelinesLogpushResourceNames(tenant, logicalName, env);
612
+ const pl = (await api.pipelineListAll()).find((p) => p.name === names.pipelineName);
613
+ if (pl) try {
614
+ await api.pipelineDelete(pl.id);
615
+ console.log(`Pipelines: destroy reconcile — removed SQL pipeline "${names.pipelineName}" [id ${pl.id}].`);
616
+ } catch (e) {
617
+ console.warn(`Pipelines: destroy reconcile — SQL pipeline "${names.pipelineName}":`, e instanceof Error ? e.message : e);
618
+ }
619
+ const sk = (await api.pipelinesV1SinkListAll()).find((s) => s.name === names.sinkName);
620
+ if (sk) try {
621
+ await deletePipelinesV1SinkWithLookup(api, sk.id, sk.name);
622
+ console.log(`Pipelines: destroy reconcile — removed sink "${names.sinkName}" [id ${sk.id}].`);
623
+ } catch (e) {
624
+ console.warn(`Pipelines: destroy reconcile — sink "${names.sinkName}":`, e instanceof Error ? e.message : e);
625
+ }
626
+ const st = (await api.pipelinesV1StreamListAll()).find((s) => s.name === names.streamName);
627
+ if (st) try {
628
+ await deletePipelinesV1StreamWithLookup(api, st.id, st.name);
629
+ console.log(`Pipelines: destroy reconcile — removed stream "${names.streamName}" [id ${st.id}].`);
630
+ } catch (e) {
631
+ console.warn(`Pipelines: destroy reconcile — stream "${names.streamName}":`, e instanceof Error ? e.message : e);
632
+ }
633
+ const cat = opts?.catalogBucketDerivedName?.trim();
634
+ if (cat) await pipelinesAutoTeardownR2DataCatalog(api, cat);
635
+ }
636
+
637
+ //#endregion
638
+ //#region src/features/logpush-job/logpush-job.desired.ts
639
+ function resolvedFilter(config, templateFilter) {
640
+ const explicit = config.filter?.trim();
641
+ if (explicit) return explicit;
642
+ const t = templateFilter?.trim();
643
+ if (t) return t;
644
+ }
645
+ /**
646
+ * Merge `output_options` from a template job (dashboard bootstrap) with
647
+ * optional `fieldNames` override from config.
648
+ */
649
+ function mergeLogpushOutputOptions(config, template) {
650
+ const baseline = template && typeof template === "object" && !Array.isArray(template) ? { ...template } : {};
651
+ if (config.fieldNames?.length) {
652
+ baseline.field_names = config.fieldNames;
653
+ return baseline;
654
+ }
655
+ if (!template || typeof template !== "object") {
656
+ baseline.field_names = [...DEFAULT_WORKERS_TRACE_LOG_FIELD_NAMES];
657
+ return baseline;
658
+ }
659
+ if (!Array.isArray(baseline.field_names)) baseline.field_names = [...DEFAULT_WORKERS_TRACE_LOG_FIELD_NAMES];
660
+ return baseline;
661
+ }
662
+ async function resolveLogpushJobDesired(config, accountId, state, api, tenantId, env) {
663
+ if (config.destinationConfEnv) {
664
+ const raw = process.env[config.destinationConfEnv]?.trim();
665
+ if (!raw) throw new Error(`logpushJobs.${config.logicalName}: environment variable "${config.destinationConfEnv}" is empty (set it to the full Logpush destination_conf, or remove this logpushJobs entry)`);
666
+ const filter$1 = resolvedFilter(config, void 0);
667
+ const output_options$1 = mergeLogpushOutputOptions(config, void 0);
668
+ return {
669
+ destination_conf: raw,
670
+ ...filter$1 ? { filter: filter$1 } : {},
671
+ output_options: output_options$1
672
+ };
673
+ }
674
+ if (config.pipelinesAuto) {
675
+ const pkey = logpushPipelinesStateKey(tenantId, config.logicalName, env);
676
+ const lpp = state.get(pkey);
677
+ if (!lpp || lpp.type !== "logpush_pipelines") throw new Error(`logpushJobs.${config.logicalName}: pipelinesAuto graph not in state — \`tamer apply\` provisions stream/sink/pipeline first`);
678
+ const token = lpp.mintedPipelinesSendTokenValue?.trim();
679
+ if (!token) throw new Error(`logpushJobs.${config.logicalName}: pipelinesAuto minted Pipelines Send token missing from state — run \`tamer apply\` to provision tokens and graph`);
680
+ const destination_conf$1 = buildPipelinesIngestDestinationConf({
681
+ streamId: lpp.streamId,
682
+ pipelineId: lpp.pipelineId,
683
+ bearerToken: token,
684
+ ingestBaseUrl: lpp.streamIngestBaseUrl
685
+ });
686
+ const filter$1 = resolvedFilter(config, void 0);
687
+ const output_options$1 = mergeLogpushOutputOptions(config, void 0);
688
+ return {
689
+ destination_conf: destination_conf$1,
690
+ ...filter$1 ? { filter: filter$1 } : {},
691
+ output_options: output_options$1
692
+ };
693
+ }
694
+ if (config.pipelinesIngest) {
695
+ const pi = config.pipelinesIngest;
696
+ const token = process.env[pi.bearerTokenEnv]?.trim();
697
+ if (!token) throw new Error(`logpushJobs.${config.logicalName}: environment variable "${pi.bearerTokenEnv}" is empty (Pipelines stream ingest Bearer token for Logpush)`);
698
+ let destination_conf$1;
699
+ try {
700
+ destination_conf$1 = buildPipelinesIngestDestinationConf({
701
+ streamId: pi.streamId,
702
+ pipelineId: pi.pipelineId,
703
+ bearerToken: token
704
+ });
705
+ } catch (e) {
706
+ const msg = e instanceof Error ? e.message : String(e);
707
+ throw new Error(`logpushJobs.${config.logicalName}: ${msg}`);
708
+ }
709
+ const filter$1 = resolvedFilter(config, void 0);
710
+ const output_options$1 = mergeLogpushOutputOptions(config, void 0);
711
+ return {
712
+ destination_conf: destination_conf$1,
713
+ ...filter$1 ? { filter: filter$1 } : {},
714
+ output_options: output_options$1
715
+ };
716
+ }
717
+ let sourceJobId;
718
+ if (config.destinationConfFromJobId != null) sourceJobId = config.destinationConfFromJobId;
719
+ else if (config.destinationConfFromJobIdEnv) {
720
+ const raw = process.env[config.destinationConfFromJobIdEnv]?.trim();
721
+ if (!raw) throw new Error(`logpushJobs.${config.logicalName}: environment variable "${config.destinationConfFromJobIdEnv}" is empty (set it to the numeric Cloudflare Logpush job id whose destination_conf should be copied)`);
722
+ const n = Number(raw);
723
+ if (!Number.isInteger(n) || n <= 0) throw new Error(`logpushJobs.${config.logicalName}: "${config.destinationConfFromJobIdEnv}" must be a positive integer job id (got "${raw}")`);
724
+ sourceJobId = n;
725
+ }
726
+ if (sourceJobId != null) {
727
+ const src = await api.logpushAccountJobGet(sourceJobId);
728
+ const conf = src.destination_conf?.trim();
729
+ if (!conf) throw new Error(`logpushJobs.${config.logicalName}: Logpush job ${sourceJobId} has no destination_conf`);
730
+ if (src.dataset && src.dataset !== config.dataset) throw new Error(`logpushJobs.${config.logicalName}: source job ${sourceJobId} dataset is "${src.dataset}" but this job declares "${config.dataset}"`);
731
+ const filter$1 = resolvedFilter(config, src.filter);
732
+ const templateOpts = src.output_options;
733
+ const output_options$1 = mergeLogpushOutputOptions(config, templateOpts);
734
+ return {
735
+ destination_conf: conf,
736
+ ...filter$1 ? { filter: filter$1 } : {},
737
+ output_options: output_options$1
738
+ };
739
+ }
740
+ const r2 = config.r2;
741
+ if (!r2) throw new Error(`logpushJobs.${config.logicalName}: internal: no destination source (expected loader to enforce r2 | pipelinesIngest | pipelinesAuto | env | job id)`);
742
+ const bucket = findR2BucketDerivedName(state, r2.bucketLogicalName);
743
+ if (!bucket) throw new Error(`logpushJobs.${config.logicalName}: R2 bucket logical "${r2.bucketLogicalName}" is not in state — run \`tamer apply\` so the bucket is created first, then re-run apply`);
744
+ const ak = process.env[r2.accessKeyIdEnv]?.trim();
745
+ const sk = process.env[r2.secretAccessKeyEnv]?.trim();
746
+ if (!ak || !sk) throw new Error(`logpushJobs.${config.logicalName}: set ${r2.accessKeyIdEnv} and ${r2.secretAccessKeyEnv} (R2 S3-compatible credentials for Logpush)`);
747
+ const destination_conf = buildR2WorkersTraceDestinationConf({
748
+ bucketDerivedName: bucket,
749
+ pathPrefix: r2.pathPrefix ?? "workers-trace-events",
750
+ accountId,
751
+ accessKeyId: ak,
752
+ secretAccessKey: sk
753
+ });
754
+ const filter = resolvedFilter(config, void 0);
755
+ const output_options = mergeLogpushOutputOptions(config, void 0);
756
+ return {
757
+ destination_conf,
758
+ ...filter ? { filter } : {},
759
+ output_options
760
+ };
761
+ }
762
+
763
+ //#endregion
764
+ //#region src/features/logpush-job/logpush-job.diff.ts
765
+ function normalizeLogpushOutputOptions(o) {
766
+ if (!o || typeof o !== "object" || Array.isArray(o)) return o;
767
+ const x = { ...o };
768
+ if (Array.isArray(x.field_names)) x.field_names = [...x.field_names].sort();
769
+ return x;
770
+ }
771
+ function normLogpushFilter(f) {
772
+ return f?.trim() || void 0;
773
+ }
774
+ /** Build `filter` for PUT when it must change (including clear). */
775
+ function logpushFilterPutBody(desired, live) {
776
+ const d = normLogpushFilter(desired.filter);
777
+ if (d === normLogpushFilter(live.filter)) return {};
778
+ return { filter: d ?? "" };
779
+ }
780
+ function logpushJobFieldChanges(desired, live, enabledDesired) {
781
+ const changes = [];
782
+ const dc = desired.destination_conf.trim();
783
+ const lc = (live.destination_conf ?? "").trim();
784
+ if (dc !== lc) changes.push({
785
+ field: "destination_conf",
786
+ from: lc,
787
+ to: dc,
788
+ kind: "mutable"
789
+ });
790
+ const df = normLogpushFilter(desired.filter);
791
+ const lf = normLogpushFilter(live.filter);
792
+ if (df !== lf) changes.push({
793
+ field: "filter",
794
+ from: lf ?? "(none)",
795
+ to: df ?? "(none)",
796
+ kind: "mutable"
797
+ });
798
+ const doo = JSON.stringify(normalizeLogpushOutputOptions(desired.output_options));
799
+ const loo = JSON.stringify(normalizeLogpushOutputOptions(live.output_options ?? {}));
800
+ if (doo !== loo) changes.push({
801
+ field: "output_options",
802
+ from: JSON.parse(loo),
803
+ to: JSON.parse(doo),
804
+ kind: "mutable"
805
+ });
806
+ const enabledLive = live.enabled !== false;
807
+ if (enabledLive !== enabledDesired) changes.push({
808
+ field: "enabled",
809
+ from: enabledLive,
810
+ to: enabledDesired,
811
+ kind: "mutable"
812
+ });
813
+ return changes;
814
+ }
815
+ async function logpushJobDiffPlanItems(args) {
816
+ const { resources, tenant, env, state, api, accountId } = args;
817
+ if (env === "local") return [];
818
+ const items = [];
819
+ for (const config of resources) {
820
+ if (config.dataset !== "workers_trace_events") continue;
821
+ const key = logpushJobStateKey(tenant.id, config.logicalName, env);
822
+ const entry = state.get(key);
823
+ if (!entry || entry.type !== "logpush_job") continue;
824
+ if (config.pipelinesAuto) {
825
+ const pkey = logpushPipelinesStateKey(tenant.id, config.logicalName, env);
826
+ const lpp = state.get(pkey);
827
+ if (!lpp || lpp.type !== "logpush_pipelines") continue;
828
+ }
829
+ const cfName = logpushJobCloudflareName(config, tenant, env);
830
+ let liveFull;
831
+ try {
832
+ liveFull = await api.logpushAccountJobGet(entry.cfJobId);
833
+ } catch {
834
+ continue;
835
+ }
836
+ const desired = await resolveLogpushJobDesired(config, accountId, state, api, tenant.id, env);
837
+ const enabledDesired = config.enabled !== false;
838
+ const changes = logpushJobFieldChanges(desired, {
839
+ destination_conf: liveFull.destination_conf,
840
+ filter: liveFull.filter,
841
+ output_options: liveFull.output_options,
842
+ enabled: liveFull.enabled
843
+ }, enabledDesired);
844
+ if (changes.length === 0) continue;
845
+ items.push({
846
+ kind: "logpush_job",
847
+ action: "update",
848
+ logicalName: config.logicalName,
849
+ derivedName: cfName,
850
+ detail: changes.map((c) => `${c.field}`).join(", "),
851
+ changes
852
+ });
853
+ }
854
+ return items;
855
+ }
856
+
857
+ //#endregion
858
+ //#region src/features/logpush-job/logpush-job.apply.ts
859
+ /** Logpush validates the Pipelines ingest URL; new streams can return DNS 1002 until the hostname propagates. */
860
+ function isRetriableLogpushIngestDnsError(message) {
861
+ return /\bDNSError\b/i.test(message) && /\b1002\b/.test(message);
862
+ }
863
+ async function logpushAccountJobCreateWithIngestRetry(api, body) {
864
+ const maxAttempts = 8;
865
+ let lastError;
866
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) try {
867
+ return await api.logpushAccountJobCreate(body);
868
+ } catch (e) {
869
+ lastError = e;
870
+ const msg = e instanceof Error ? e.message : String(e);
871
+ if (attempt === maxAttempts || !isRetriableLogpushIngestDnsError(msg)) throw e;
872
+ const waitMs = Math.min(3e4, 1500 * 2 ** (attempt - 1));
873
+ console.warn(`Logpush create "${body.name}": destination validation error, retry ${attempt}/${maxAttempts} in ${waitMs}ms (ingest DNS may lag right after a new Pipelines stream). ${msg.slice(0, 240)}`);
874
+ await new Promise((r) => setTimeout(r, waitMs));
875
+ }
876
+ throw lastError;
877
+ }
878
+ function writeState(state, key, config, cfName, cfJobId, dataset, existingCreatedAt) {
879
+ const ts = (/* @__PURE__ */ new Date()).toISOString();
880
+ state.set(key, {
881
+ type: "logpush_job",
882
+ logicalName: config.logicalName,
883
+ derivedName: cfName,
884
+ cfJobId,
885
+ dataset,
886
+ createdAt: existingCreatedAt ?? ts,
887
+ updatedAt: ts
888
+ });
889
+ }
890
+ async function reconcileLogpushJob(config, cfJobId, cfName, accountId, tenant, env, state, api, key, st) {
891
+ const fullLive = await api.logpushAccountJobGet(cfJobId);
892
+ const desired = await resolveLogpushJobDesired(config, accountId, state, api, tenant.id, env);
893
+ const enabledDesired = config.enabled !== false;
894
+ const changes = logpushJobFieldChanges(desired, {
895
+ destination_conf: fullLive.destination_conf,
896
+ filter: fullLive.filter,
897
+ output_options: fullLive.output_options,
898
+ enabled: fullLive.enabled
899
+ }, enabledDesired);
900
+ if (changes.length === 0) return;
901
+ await api.logpushAccountJobUpdate(cfJobId, {
902
+ destination_conf: desired.destination_conf,
903
+ enabled: enabledDesired,
904
+ output_options: desired.output_options,
905
+ ...logpushFilterPutBody(desired, fullLive)
906
+ });
907
+ const ts = (/* @__PURE__ */ new Date()).toISOString();
908
+ state.set(key, {
909
+ ...st,
910
+ updatedAt: ts
911
+ });
912
+ console.log(`Updated Logpush job "${cfName}" (${config.dataset}) [id ${cfJobId}]: ${changes.map((c) => c.field).join(", ")}`);
913
+ }
914
+ async function logpushJobApply(resources, tenant, env, accountId, api, state) {
915
+ if (resources.length === 0 || env === "local") return;
916
+ const jobs = await api.logpushAccountJobsList();
917
+ const byNameDataset = /* @__PURE__ */ new Map();
918
+ for (const j of jobs) if (j.name && j.dataset) byNameDataset.set(`${j.dataset}\0${j.name}`, j);
919
+ for (const config of resources) {
920
+ if (config.dataset !== "workers_trace_events") throw new Error(`logpushJobs.${config.logicalName}: unsupported dataset "${config.dataset}" (only workers_trace_events)`);
921
+ if (config.pipelinesAuto) await ensurePipelinesLogpushProvision(config, config.pipelinesAuto, tenant, env, accountId, state, api);
922
+ const key = logpushJobStateKey(tenant.id, config.logicalName, env);
923
+ const cfName = logpushJobCloudflareName(config, tenant, env);
924
+ const mapKey = `${config.dataset}\0${cfName}`;
925
+ const existingState = state.get(key);
926
+ if (existingState?.type === "logpush_job") {
927
+ if (jobs.find((j) => j.id === existingState.cfJobId)) {
928
+ await reconcileLogpushJob(config, existingState.cfJobId, cfName, accountId, tenant, env, state, api, key, existingState);
929
+ continue;
930
+ }
931
+ }
932
+ const hit = byNameDataset.get(mapKey);
933
+ if (hit?.id != null) {
934
+ writeState(state, key, config, cfName, hit.id, config.dataset, existingState?.type === "logpush_job" ? existingState.createdAt : void 0);
935
+ console.log(`Logpush job "${cfName}" (${config.dataset}) already exists [id ${hit.id}] — recorded in state.`);
936
+ const st = state.get(key);
937
+ if (st?.type === "logpush_job") await reconcileLogpushJob(config, hit.id, cfName, accountId, tenant, env, state, api, key, st);
938
+ continue;
939
+ }
940
+ const desired = await resolveLogpushJobDesired(config, accountId, state, api, tenant.id, env);
941
+ const created = await logpushAccountJobCreateWithIngestRetry(api, {
942
+ name: cfName,
943
+ dataset: config.dataset,
944
+ destination_conf: desired.destination_conf,
945
+ enabled: config.enabled !== false,
946
+ ...logpushFilterPutBody(desired, {}),
947
+ output_options: desired.output_options
948
+ });
949
+ writeState(state, key, config, cfName, created.id, config.dataset, existingState?.type === "logpush_job" ? existingState.createdAt : void 0);
950
+ console.log(`Created Logpush job "${cfName}" (${config.dataset}) [id ${created.id}].`);
951
+ }
952
+ }
953
+
954
+ //#endregion
955
+ //#region src/features/logpush-job/logpush-job.sync.ts
956
+ async function logpushJobSync(resources, tenant, env, api, state) {
957
+ if (resources.length === 0 || env === "local") return;
958
+ const jobs = await api.logpushAccountJobsList();
959
+ for (const config of resources) {
960
+ const cfName = logpushJobCloudflareName(config, tenant, env);
961
+ const hit = jobs.find((j) => j.name === cfName && j.dataset === config.dataset);
962
+ if (!hit?.id) continue;
963
+ const key = logpushJobStateKey(tenant.id, config.logicalName, env);
964
+ const existing = state.get(key);
965
+ const ts = (/* @__PURE__ */ new Date()).toISOString();
966
+ state.set(key, {
967
+ type: "logpush_job",
968
+ logicalName: config.logicalName,
969
+ derivedName: cfName,
970
+ cfJobId: hit.id,
971
+ dataset: config.dataset,
972
+ createdAt: existing?.type === "logpush_job" ? existing.createdAt : ts,
973
+ updatedAt: ts
974
+ });
975
+ }
976
+ }
977
+
978
+ //#endregion
979
+ //#region src/features/logpush-job/logpush-job.drift.ts
980
+ function logpushJobDrift(allJobs, resources, env, tenant, state) {
981
+ const drift = {
982
+ kind: "logpush_job",
983
+ missingFromCloudflare: [],
984
+ unrecordedInState: [],
985
+ undeployed: []
986
+ };
987
+ for (const config of resources) {
988
+ const cfName = logpushJobCloudflareName(config, tenant, env);
989
+ const key = logpushJobStateKey(tenant.id, config.logicalName, env);
990
+ const tracked = state.get(key);
991
+ const st = tracked?.type === "logpush_job" ? tracked : void 0;
992
+ const onCf = allJobs.find((j) => j.name === cfName && j.dataset === config.dataset);
993
+ const idStillThere = st && allJobs.some((j) => j.id === st.cfJobId);
994
+ if (st && !idStillThere) drift.missingFromCloudflare.push({
995
+ logicalName: config.logicalName,
996
+ derivedName: cfName,
997
+ cfId: String(st.cfJobId)
998
+ });
999
+ else if (onCf && !st) drift.unrecordedInState.push({
1000
+ logicalName: config.logicalName,
1001
+ derivedName: cfName,
1002
+ cfId: String(onCf.id)
1003
+ });
1004
+ else if (!onCf && !st) drift.undeployed.push({
1005
+ logicalName: config.logicalName,
1006
+ derivedName: cfName
1007
+ });
1008
+ }
1009
+ return drift;
1010
+ }
1011
+
1012
+ //#endregion
1013
+ //#region src/features/logpush-job/logpush-job.destroy.ts
1014
+ /**
1015
+ * `logpush_pipelines:t-${tenantId}-${logicalName}-${env}` (logical name may
1016
+ * contain hyphens, so we match prefix+suffix to scope to this stack).
1017
+ */
1018
+ function isLogpushPipelinesKeyForStack(key, tenantId, env) {
1019
+ const prefix = `logpush_pipelines:t-${tenantId}-`;
1020
+ const suffix = `-${env}`;
1021
+ return key.startsWith(prefix) && key.endsWith(suffix);
1022
+ }
1023
+ /**
1024
+ * Delete Logpush jobs declared in config and remove their state rows.
1025
+ * Runs before R2 teardown so the job stops writing to the bucket.
1026
+ * For `pipelinesAuto`, deletes the Logpush job first, then stream / sink / SQL pipeline.
1027
+ *
1028
+ * Also sweeps any remaining `logpush_pipelines` state for this stack (e.g. config
1029
+ * no longer lists logpush, or the job row was lost while pipelines state remained).
1030
+ */
1031
+ async function logpushJobDestroy(env, state, api, config) {
1032
+ if (env === "local") return;
1033
+ const jobs = getLogpushJobs(config);
1034
+ const allowed = new Set(jobs.map((j) => j.logicalName));
1035
+ const autoByLogical = new Map(jobs.filter((j) => j.pipelinesAuto).map((j) => [j.logicalName, j]));
1036
+ for (const [key, entry] of Object.entries(state.getAll())) {
1037
+ if (entry.type !== "logpush_job") continue;
1038
+ const job = entry;
1039
+ if (allowed.size > 0 && !allowed.has(job.logicalName)) continue;
1040
+ try {
1041
+ await api.logpushAccountJobDelete(job.cfJobId);
1042
+ state.delete(key);
1043
+ console.log(`Deleted Logpush job "${job.derivedName}" [id ${job.cfJobId}].`);
1044
+ } catch (err) {
1045
+ console.warn(`Failed to delete Logpush job ${job.derivedName} [id ${job.cfJobId}]:`, err instanceof Error ? err.message : err);
1046
+ }
1047
+ if (autoByLogical.has(job.logicalName)) {
1048
+ const pkey = logpushPipelinesStateKey(config.tenant.id, job.logicalName, env);
1049
+ const lpp = state.get(pkey);
1050
+ if (lpp?.type === "logpush_pipelines") {
1051
+ await deletePipelinesLogpushGraph(api, lpp);
1052
+ state.delete(pkey);
1053
+ }
1054
+ }
1055
+ }
1056
+ for (const [key, entry] of Object.entries(state.getAll())) {
1057
+ if (entry.type !== "logpush_pipelines") continue;
1058
+ if (!isLogpushPipelinesKeyForStack(key, config.tenant.id, env)) continue;
1059
+ const lpp = entry;
1060
+ try {
1061
+ await deletePipelinesLogpushGraph(api, lpp);
1062
+ state.delete(key);
1063
+ } catch (err) {
1064
+ console.warn(`Failed to destroy Pipelines graph for [${key}]:`, err instanceof Error ? err.message : err);
1065
+ }
1066
+ }
1067
+ for (const j of jobs) {
1068
+ if (!j.pipelinesAuto) continue;
1069
+ const catBucket = findR2BucketDerivedName(state, j.pipelinesAuto.catalogBucketLogicalName);
1070
+ try {
1071
+ await reconcilePipelinesAutoCloudResourcesByName(api, config.tenant, j.logicalName, env, catBucket ? { catalogBucketDerivedName: catBucket } : void 0);
1072
+ } catch (e) {
1073
+ console.warn(`Pipelines: destroy reconcile (SQL/sink/stream) for logpush ${j.logicalName}:`, e instanceof Error ? e.message : e);
1074
+ }
1075
+ try {
1076
+ await reconcilePipelinesAutoMintedAccountTokensByName(api, config.tenant, j.logicalName, env);
1077
+ } catch (e) {
1078
+ console.warn(`Pipelines: destroy reconcile (account API tokens) for logpush ${j.logicalName}:`, e instanceof Error ? e.message : e);
1079
+ }
1080
+ }
1081
+ }
1082
+
1083
+ //#endregion
1084
+ //#region src/features/logpush-job/logpush-job.status.ts
1085
+ function logpushJobStatus(resources, tenant, env, state, liveJobs) {
1086
+ const rows = [];
1087
+ for (const config of resources) {
1088
+ const key = logpushJobStateKey(tenant.id, config.logicalName, env);
1089
+ const entry = state.get(key);
1090
+ const st = entry?.type === "logpush_job" ? entry : void 0;
1091
+ const live = liveJobs && st?.cfJobId != null ? liveJobs.find((j) => j.id === st.cfJobId) : void 0;
1092
+ rows.push({
1093
+ logicalName: config.logicalName,
1094
+ jobName: st?.derivedName ?? "(unknown)",
1095
+ cfJobId: st?.cfJobId,
1096
+ status: st ? "ok" : "missing",
1097
+ cfEnabled: live?.enabled,
1098
+ cfError: live?.error_message
1099
+ });
1100
+ }
1101
+ return rows;
1102
+ }
1103
+
1104
+ //#endregion
1105
+ export { logpushJobApply as a, logpushJobSync as i, logpushJobDestroy as n, logpushJobDiffPlanItems as o, logpushJobDrift as r, logpushJobStateKey as s, logpushJobStatus as t };
1106
+ //# sourceMappingURL=logpush-job-xS7270FZ.mjs.map