@useairfoil/connector-kit 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +201 -0
- package/METRICS.md +38 -42
- package/README.md +111 -444
- package/dist/annotations-vzSzXtr-.d.mts +34 -0
- package/dist/arrow-flight-DDQ0ktx9.mjs +251 -0
- package/dist/arrow-flight-DDQ0ktx9.mjs.map +1 -0
- package/dist/auth/index.d.mts +2 -0
- package/dist/{auth-DvU_CZ7C.js → auth/index.mjs} +6 -8
- package/dist/auth/index.mjs.map +1 -0
- package/dist/connector-app/index.d.mts +2 -0
- package/dist/connector-app/index.mjs +2 -0
- package/dist/connector-app-C6_dZnhE.mjs +256 -0
- package/dist/connector-app-C6_dZnhE.mjs.map +1 -0
- package/dist/{constants-D113Ut1x.js → constants-v3UX-hf6.mjs} +5 -5
- package/dist/constants-v3UX-hf6.mjs.map +1 -0
- package/dist/demo/index.d.mts +2 -0
- package/dist/demo/index.mjs +2 -0
- package/dist/demo-DqTJ7TWI.mjs +204 -0
- package/dist/demo-DqTJ7TWI.mjs.map +1 -0
- package/dist/errors/index.d.mts +2 -0
- package/dist/errors/index.mjs +2 -0
- package/dist/{errors-DHWylqRZ.js → errors-pT0YVJU5.mjs} +2 -3
- package/dist/errors-pT0YVJU5.mjs.map +1 -0
- package/dist/iceberg/index.d.mts +3 -0
- package/dist/iceberg/index.mjs +3 -0
- package/dist/iceberg-Cj2PwlNm.mjs +30 -0
- package/dist/iceberg-Cj2PwlNm.mjs.map +1 -0
- package/dist/index-BTAbexBp.d.mts +55 -0
- package/dist/{index-DXa26Ry5.d.ts → index-BX8G7d41.d.mts} +1 -2
- package/dist/{index-VpHaZXZw.d.ts → index-Byr8NTdG.d.mts} +11 -13
- package/dist/index-CV6V6fyx.d.mts +79 -0
- package/dist/{index-C5Ma7V7t.d.ts → index-CZKx2Yce.d.mts} +72 -37
- package/dist/{index-DYpEMJf-.d.ts → index-CgvFomO6.d.mts} +6 -7
- package/dist/{index-bfjH7Yb0.d.ts → index-DeTwIUKl.d.mts} +9 -10
- package/dist/{index-DKoRwyFn.d.ts → index-DkVNO6ks.d.mts} +2 -5
- package/dist/{index-BKw7jA2a.d.ts → index-dHS59EZl.d.mts} +10 -8
- package/dist/{index-CR9HJDdr.d.ts → index-eFuoMZZ3.d.mts} +18 -20
- package/dist/{index-ZSuiZX5H.d.ts → index-fHIJJoRe.d.mts} +25 -30
- package/dist/index-uFa2wwYo.d.mts +27 -0
- package/dist/index.d.mts +16 -0
- package/dist/index.mjs +45 -0
- package/dist/index.mjs.map +1 -0
- package/dist/ingestion/index.d.mts +2 -0
- package/dist/ingestion/index.mjs +2 -0
- package/dist/ingestion-DoCUwm5N.mjs +275 -0
- package/dist/ingestion-DoCUwm5N.mjs.map +1 -0
- package/dist/ingestor/index.d.mts +3 -0
- package/dist/ingestor/index.mjs +4 -0
- package/dist/ingestor-DjCEzBzZ.mjs +26 -0
- package/dist/ingestor-DjCEzBzZ.mjs.map +1 -0
- package/dist/instrumented-Bq1WX2L5.mjs +29 -0
- package/dist/instrumented-Bq1WX2L5.mjs.map +1 -0
- package/dist/manifest/index.d.mts +2 -0
- package/dist/manifest/{index.js → index.mjs} +2 -4
- package/dist/{manifest-P7kBnch9.js → manifest-CABeIhAG.mjs} +14 -19
- package/dist/manifest-CABeIhAG.mjs.map +1 -0
- package/dist/{telemetry-j_nxwYfh.js → metrics-C901IIYZ.mjs} +172 -26
- package/dist/metrics-C901IIYZ.mjs.map +1 -0
- package/dist/{metrics-C4V1pm6v.d.ts → metrics-DoJ0WeKu.d.mts} +18 -13
- package/dist/metrics.d.mts +2 -0
- package/dist/metrics.mjs +2 -0
- package/dist/{chunk-Bp6m_JJh.js → rolldown-runtime-D7D4PA-g.mjs} +4 -4
- package/dist/runtime-config/index.d.mts +2 -0
- package/dist/runtime-config/index.mjs +3 -0
- package/dist/runtime-config-Bn8Fivt1.mjs +100 -0
- package/dist/runtime-config-Bn8Fivt1.mjs.map +1 -0
- package/dist/server-D1TTN0_D.mjs +144 -0
- package/dist/server-D1TTN0_D.mjs.map +1 -0
- package/dist/service-BRoy_EId.d.mts +27 -0
- package/dist/service-jWxxfEht.mjs +7 -0
- package/dist/service-jWxxfEht.mjs.map +1 -0
- package/dist/state-store/index.d.mts +2 -0
- package/dist/state-store/index.mjs +2 -0
- package/dist/{state-store-BdYulQOY.js → state-store-DhSrKVEH.mjs} +15 -22
- package/dist/state-store-DhSrKVEH.mjs.map +1 -0
- package/dist/status/{index.d.ts → index.d.mts} +1 -4
- package/dist/status/{index.js → index.mjs} +2 -6
- package/dist/{status-BTTd7AUR.js → status-4h6Oxh-O.mjs} +14 -16
- package/dist/status-4h6Oxh-O.mjs.map +1 -0
- package/dist/table-CxM09xf7.mjs +307 -0
- package/dist/table-CxM09xf7.mjs.map +1 -0
- package/dist/types-NtSv6gwl.d.mts +220 -0
- package/dist/webhook/index.d.mts +3 -0
- package/dist/webhook/index.mjs +12 -0
- package/dist/webhook/index.mjs.map +1 -0
- package/package.json +72 -58
- package/src/catalog/retry.ts +25 -0
- package/src/catalog/table.ts +237 -0
- package/src/connector-app/create-table.ts +47 -0
- package/src/connector-app/demo.ts +47 -0
- package/src/connector-app/index.ts +3 -0
- package/src/connector-app/setup.ts +69 -0
- package/src/connector-app/start.ts +1 -1
- package/src/core/builder.ts +15 -23
- package/src/core/index.ts +11 -1
- package/src/core/types.ts +114 -39
- package/src/demo/definition.ts +68 -0
- package/src/demo/generate.ts +167 -0
- package/src/demo/index.ts +3 -0
- package/src/demo/run.ts +172 -0
- package/src/iceberg/annotations.ts +49 -0
- package/src/iceberg/error.ts +9 -0
- package/src/iceberg/index.ts +3 -0
- package/src/iceberg/table.ts +461 -0
- package/src/index.ts +3 -1
- package/src/ingestion/engine.ts +240 -60
- package/src/ingestor/arrow-flight.ts +141 -0
- package/src/ingestor/arrow.ts +248 -0
- package/src/ingestor/console.ts +13 -0
- package/src/ingestor/index.ts +13 -0
- package/src/ingestor/instrumented.ts +61 -0
- package/src/ingestor/service.ts +30 -0
- package/src/manifest/config.ts +3 -3
- package/src/metrics.ts +66 -20
- package/src/runtime-config/config.ts +11 -8
- package/src/runtime-config/constants.ts +3 -2
- package/src/runtime-config/index.ts +2 -7
- package/src/runtime-config/layer.ts +1 -1
- package/src/runtime-config/table-bindings.ts +50 -136
- package/src/state-store/layer-key-value-store.ts +8 -3
- package/src/state-store/layer-memory.ts +1 -1
- package/src/state-store/layer-sql.ts +4 -4
- package/src/state-store/schema.ts +2 -2
- package/src/state-store/state.ts +14 -2
- package/src/status/route.ts +2 -2
- package/src/telemetry.ts +20 -28
- package/src/webhook/server.ts +39 -40
- package/dist/auth/index.d.ts +0 -2
- package/dist/auth/index.js +0 -3
- package/dist/auth-DvU_CZ7C.js.map +0 -1
- package/dist/connector-app/index.d.ts +0 -14
- package/dist/connector-app/index.js +0 -12
- package/dist/connector-app-BljHd0Pe.js +0 -64
- package/dist/connector-app-BljHd0Pe.js.map +0 -1
- package/dist/constants-D113Ut1x.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.js +0 -3
- package/dist/errors-DHWylqRZ.js.map +0 -1
- package/dist/index-BA8ASHKr.d.ts +0 -31
- package/dist/index.d.ts +0 -14
- package/dist/index.js +0 -60
- package/dist/index.js.map +0 -1
- package/dist/ingestion/index.d.ts +0 -6
- package/dist/ingestion/index.js +0 -11
- package/dist/ingestion-QBGKrFr9.js +0 -200
- package/dist/ingestion-QBGKrFr9.js.map +0 -1
- package/dist/manifest/index.d.ts +0 -2
- package/dist/manifest-P7kBnch9.js.map +0 -1
- package/dist/metrics-inkCAxKq.js +0 -136
- package/dist/metrics-inkCAxKq.js.map +0 -1
- package/dist/metrics.d.ts +0 -4
- package/dist/metrics.js +0 -5
- package/dist/publisher/index.d.ts +0 -6
- package/dist/publisher/index.js +0 -8
- package/dist/publisher-BdWJOfer.js +0 -138
- package/dist/publisher-BdWJOfer.js.map +0 -1
- package/dist/runtime-config/index.d.ts +0 -4
- package/dist/runtime-config/index.js +0 -5
- package/dist/runtime-config-IbyCVpei.js +0 -60
- package/dist/runtime-config-IbyCVpei.js.map +0 -1
- package/dist/server-D-eLX3yN.js +0 -183
- package/dist/server-D-eLX3yN.js.map +0 -1
- package/dist/service-CUjGhe-7.d.ts +0 -32
- package/dist/service-DiP-XETo.js +0 -8
- package/dist/service-DiP-XETo.js.map +0 -1
- package/dist/state-store/index.d.ts +0 -3
- package/dist/state-store/index.js +0 -5
- package/dist/state-store-BdYulQOY.js.map +0 -1
- package/dist/status-BTTd7AUR.js.map +0 -1
- package/dist/table-bindings-Dp1DhTeS.js +0 -106
- package/dist/table-bindings-Dp1DhTeS.js.map +0 -1
- package/dist/telemetry-j_nxwYfh.js.map +0 -1
- package/dist/types-jVD5Ys43.d.ts +0 -174
- package/dist/webhook/index.d.ts +0 -7
- package/dist/webhook/index.js +0 -9
- package/dist/webhook-MhFcLW-o.js +0 -13
- package/dist/webhook-MhFcLW-o.js.map +0 -1
- package/src/publisher/console.ts +0 -18
- package/src/publisher/index.ts +0 -7
- package/src/publisher/instrumented.ts +0 -75
- package/src/publisher/service.ts +0 -37
- package/src/publisher/wings.ts +0 -204
package/dist/metrics-inkCAxKq.js
DELETED
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
-
import { t as Attr } from "./telemetry-j_nxwYfh.js";
|
|
3
|
-
import { DateTime, Effect, Metric, Option } from "effect";
|
|
4
|
-
|
|
5
|
-
//#region src/metrics.ts
|
|
6
|
-
var metrics_exports = /* @__PURE__ */ __export({
|
|
7
|
-
apiRetries: () => apiRetries,
|
|
8
|
-
batchSize: () => batchSize,
|
|
9
|
-
batches: () => batches,
|
|
10
|
-
entitiesDeleted: () => entitiesDeleted,
|
|
11
|
-
entitiesUpserted: () => entitiesUpserted,
|
|
12
|
-
lastSuccessTimestamp: () => lastSuccessTimestamp,
|
|
13
|
-
publishDuration: () => publishDuration,
|
|
14
|
-
recordApiRetry: () => recordApiRetry,
|
|
15
|
-
recordWebhookRequest: () => recordWebhookRequest,
|
|
16
|
-
setLastSuccessTimestamp: () => setLastSuccessTimestamp,
|
|
17
|
-
setSyncState: () => setSyncState,
|
|
18
|
-
setWebhookQueueDepth: () => setWebhookQueueDepth,
|
|
19
|
-
syncState: () => syncState,
|
|
20
|
-
webhookQueueDepth: () => webhookQueueDepth,
|
|
21
|
-
webhookRequests: () => webhookRequests,
|
|
22
|
-
withResourceAttributes: () => withResourceAttributes
|
|
23
|
-
});
|
|
24
|
-
const entitiesUpserted = Metric.counter("airfoil.connector.entity.upserts", {
|
|
25
|
-
description: "Total entity upsert mutations published by connector sources",
|
|
26
|
-
incremental: true,
|
|
27
|
-
attributes: { unit: "{mutation}" }
|
|
28
|
-
});
|
|
29
|
-
const entitiesDeleted = Metric.counter("airfoil.connector.entity.deletes", {
|
|
30
|
-
description: "Total entity delete mutations published by connector sources",
|
|
31
|
-
incremental: true,
|
|
32
|
-
attributes: { unit: "{mutation}" }
|
|
33
|
-
});
|
|
34
|
-
const batches = Metric.counter("airfoil.connector.batches", {
|
|
35
|
-
description: "Total resource batches published by connector sources",
|
|
36
|
-
incremental: true,
|
|
37
|
-
attributes: { unit: "{batch}" }
|
|
38
|
-
});
|
|
39
|
-
const batchSize = Metric.histogram("airfoil.connector.batch.size", {
|
|
40
|
-
description: "Distribution of resource mutation batch sizes",
|
|
41
|
-
boundaries: [
|
|
42
|
-
1,
|
|
43
|
-
5,
|
|
44
|
-
10,
|
|
45
|
-
25,
|
|
46
|
-
50,
|
|
47
|
-
100,
|
|
48
|
-
250,
|
|
49
|
-
500,
|
|
50
|
-
1e3
|
|
51
|
-
],
|
|
52
|
-
attributes: { unit: "{mutation}" }
|
|
53
|
-
});
|
|
54
|
-
const publishDuration = Metric.timer("airfoil.connector.publish.duration", {
|
|
55
|
-
description: "Publisher call duration in milliseconds",
|
|
56
|
-
boundaries: [
|
|
57
|
-
1,
|
|
58
|
-
5,
|
|
59
|
-
10,
|
|
60
|
-
25,
|
|
61
|
-
50,
|
|
62
|
-
100,
|
|
63
|
-
250,
|
|
64
|
-
500,
|
|
65
|
-
1e3,
|
|
66
|
-
2500,
|
|
67
|
-
5e3,
|
|
68
|
-
1e4
|
|
69
|
-
],
|
|
70
|
-
attributes: { unit: "ms" }
|
|
71
|
-
});
|
|
72
|
-
const webhookRequests = Metric.counter("airfoil.connector.webhook.requests", {
|
|
73
|
-
description: "Total webhook HTTP requests handled by connector routes",
|
|
74
|
-
incremental: true,
|
|
75
|
-
attributes: { unit: "{request}" }
|
|
76
|
-
});
|
|
77
|
-
const webhookQueueDepth = Metric.gauge("airfoil.connector.webhook.queue.depth", {
|
|
78
|
-
description: "Current queued webhook batches waiting for publish",
|
|
79
|
-
attributes: { unit: "{batch}" }
|
|
80
|
-
});
|
|
81
|
-
const syncState = Metric.gauge("airfoil.connector.sync.state", {
|
|
82
|
-
description: "Connector resource sync state as a Prometheus state-set gauge",
|
|
83
|
-
attributes: { unit: "1" }
|
|
84
|
-
});
|
|
85
|
-
const lastSuccessTimestamp = Metric.gauge("airfoil.connector.last_success.timestamp", {
|
|
86
|
-
description: "Unix timestamp of the last successful durable source checkpoint",
|
|
87
|
-
attributes: { unit: "s" }
|
|
88
|
-
});
|
|
89
|
-
const apiRetries = Metric.counter("airfoil.connector.api.retries", {
|
|
90
|
-
description: "Total provider API retries performed by the connector",
|
|
91
|
-
incremental: true,
|
|
92
|
-
attributes: { unit: "{retry}" }
|
|
93
|
-
});
|
|
94
|
-
const resourceAttributes = (attrs) => ({
|
|
95
|
-
[Attr.connectorName]: attrs.connector,
|
|
96
|
-
[Attr.resourceName]: attrs.resource,
|
|
97
|
-
[Attr.resourceSource]: attrs.source
|
|
98
|
-
});
|
|
99
|
-
const syncAttributes = (attrs) => ({
|
|
100
|
-
[Attr.connectorName]: attrs.connector,
|
|
101
|
-
[Attr.resourceName]: attrs.resource
|
|
102
|
-
});
|
|
103
|
-
const recordWebhookRequest = (attrs) => Metric.update(Metric.withAttributes(webhookRequests, {
|
|
104
|
-
[Attr.connectorName]: attrs.connector,
|
|
105
|
-
[Attr.webhookPath]: attrs.path,
|
|
106
|
-
[Attr.webhookOutcome]: attrs.outcome
|
|
107
|
-
}), 1);
|
|
108
|
-
const setWebhookQueueDepth = (connector, depth) => Metric.update(Metric.withAttributes(webhookQueueDepth, { [Attr.connectorName]: connector }), depth);
|
|
109
|
-
const setLastSuccessTimestamp = (attrs, lastSuccessAt) => {
|
|
110
|
-
const timestamp = lastSuccessAt === void 0 ? 0 : DateTime.make(lastSuccessAt).pipe(Option.map((dateTime) => DateTime.toEpochMillis(dateTime) / 1e3), Option.getOrElse(() => 0));
|
|
111
|
-
return Metric.update(Metric.withAttributes(lastSuccessTimestamp, {
|
|
112
|
-
[Attr.connectorName]: attrs.connector,
|
|
113
|
-
[Attr.resourceName]: attrs.resource,
|
|
114
|
-
[Attr.resourceSource]: attrs.source
|
|
115
|
-
}), timestamp);
|
|
116
|
-
};
|
|
117
|
-
const recordApiRetry = (attrs) => Metric.update(Metric.withAttributes(apiRetries, {
|
|
118
|
-
[Attr.connectorName]: attrs.connector,
|
|
119
|
-
[Attr.apiRetryReason]: attrs.reason
|
|
120
|
-
}), 1);
|
|
121
|
-
const syncStates = [
|
|
122
|
-
"pending",
|
|
123
|
-
"backfilling",
|
|
124
|
-
"live",
|
|
125
|
-
"error"
|
|
126
|
-
];
|
|
127
|
-
const setSyncState = (attrs, current) => Effect.forEach(syncStates, (state) => Metric.update(Metric.withAttributes(syncState, {
|
|
128
|
-
...syncAttributes(attrs),
|
|
129
|
-
[Attr.syncState]: state
|
|
130
|
-
}), state === current ? 1 : 0), { discard: true });
|
|
131
|
-
/** @internal Converts the ergonomic SDK input to canonical OTel attributes. */
|
|
132
|
-
const withResourceAttributes = resourceAttributes;
|
|
133
|
-
|
|
134
|
-
//#endregion
|
|
135
|
-
export { withResourceAttributes as _, entitiesUpserted as a, publishDuration as c, setLastSuccessTimestamp as d, setSyncState as f, webhookRequests as g, webhookQueueDepth as h, entitiesDeleted as i, recordApiRetry as l, syncState as m, batchSize as n, lastSuccessTimestamp as o, setWebhookQueueDepth as p, batches as r, metrics_exports as s, apiRetries as t, recordWebhookRequest as u };
|
|
136
|
-
//# sourceMappingURL=metrics-inkCAxKq.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"metrics-inkCAxKq.js","names":[],"sources":["../src/metrics.ts"],"sourcesContent":["import { DateTime, Effect, Metric, Option } from \"effect\";\n\nimport type { SyncState } from \"./core/types\";\n\nimport { Attr } from \"./telemetry\";\n\nexport type BatchSource = \"backfill\" | \"changes\" | \"webhook\";\nexport type BatchOutcome = \"accepted\" | \"rejected\" | \"error\";\nexport type ApiRetryReason = \"transport\" | \"timeout\" | \"rate_limit\" | \"server_error\";\nexport type WebhookOutcome =\n | \"ok\"\n | \"read_error\"\n | \"invalid_json\"\n | \"invalid_payload\"\n | \"rejected\"\n | \"handler_error\";\n\nexport type ResourceMetricAttrs = {\n readonly connector: string;\n readonly resource: string;\n readonly source: BatchSource;\n};\n\nexport type WebhookMetricAttrs = {\n readonly connector: string;\n readonly path: string;\n readonly outcome: WebhookOutcome;\n};\n\nexport type SyncStateAttrs = {\n readonly connector: string;\n readonly resource: string;\n};\n\nexport type LastSuccessMetricAttrs = {\n readonly connector: string;\n readonly resource: string;\n readonly source: \"backfill\" | \"changes\";\n};\n\nexport type ApiRetryMetricAttrs = {\n readonly connector: string;\n readonly reason: ApiRetryReason;\n};\n\nexport const entitiesUpserted = Metric.counter(\"airfoil.connector.entity.upserts\", {\n description: \"Total entity upsert mutations published by connector sources\",\n incremental: true,\n attributes: { unit: \"{mutation}\" },\n});\n\nexport const entitiesDeleted = Metric.counter(\"airfoil.connector.entity.deletes\", {\n description: \"Total entity delete mutations published by connector sources\",\n incremental: true,\n attributes: { unit: \"{mutation}\" },\n});\n\nexport const batches = Metric.counter(\"airfoil.connector.batches\", {\n description: \"Total resource batches published by connector sources\",\n incremental: true,\n attributes: { unit: \"{batch}\" },\n});\n\nexport const batchSize = Metric.histogram(\"airfoil.connector.batch.size\", {\n description: \"Distribution of resource mutation batch sizes\",\n boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],\n attributes: { unit: \"{mutation}\" },\n});\n\nexport const publishDuration = Metric.timer(\"airfoil.connector.publish.duration\", {\n description: \"Publisher call duration in milliseconds\",\n boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],\n attributes: { unit: \"ms\" },\n});\n\nexport const webhookRequests = Metric.counter(\"airfoil.connector.webhook.requests\", {\n description: \"Total webhook HTTP requests handled by connector routes\",\n incremental: true,\n attributes: { unit: \"{request}\" },\n});\n\nexport const webhookQueueDepth = Metric.gauge(\"airfoil.connector.webhook.queue.depth\", {\n description: \"Current queued webhook batches waiting for publish\",\n attributes: { unit: \"{batch}\" },\n});\n\nexport const syncState = Metric.gauge(\"airfoil.connector.sync.state\", {\n description: \"Connector resource sync state as a Prometheus state-set gauge\",\n attributes: { unit: \"1\" },\n});\n\nexport const lastSuccessTimestamp = Metric.gauge(\"airfoil.connector.last_success.timestamp\", {\n description: \"Unix timestamp of the last successful durable source checkpoint\",\n attributes: { unit: \"s\" },\n});\n\nexport const apiRetries = Metric.counter(\"airfoil.connector.api.retries\", {\n description: \"Total provider API retries performed by the connector\",\n incremental: true,\n attributes: { unit: \"{retry}\" },\n});\n\nconst resourceAttributes = (attrs: ResourceMetricAttrs) => ({\n [Attr.connectorName]: attrs.connector,\n [Attr.resourceName]: attrs.resource,\n [Attr.resourceSource]: attrs.source,\n});\n\nconst syncAttributes = (attrs: SyncStateAttrs) => ({\n [Attr.connectorName]: attrs.connector,\n [Attr.resourceName]: attrs.resource,\n});\n\nexport const recordWebhookRequest = (attrs: WebhookMetricAttrs) =>\n Metric.update(\n Metric.withAttributes(webhookRequests, {\n [Attr.connectorName]: attrs.connector,\n [Attr.webhookPath]: attrs.path,\n [Attr.webhookOutcome]: attrs.outcome,\n }),\n 1,\n );\n\nexport const setWebhookQueueDepth = (connector: string, depth: number) =>\n Metric.update(\n Metric.withAttributes(webhookQueueDepth, { [Attr.connectorName]: connector }),\n depth,\n );\n\nexport const setLastSuccessTimestamp = (attrs: LastSuccessMetricAttrs, lastSuccessAt?: string) => {\n const timestamp =\n lastSuccessAt === undefined\n ? 0\n : DateTime.make(lastSuccessAt).pipe(\n Option.map((dateTime) => DateTime.toEpochMillis(dateTime) / 1000),\n Option.getOrElse(() => 0),\n );\n return Metric.update(\n Metric.withAttributes(lastSuccessTimestamp, {\n [Attr.connectorName]: attrs.connector,\n [Attr.resourceName]: attrs.resource,\n [Attr.resourceSource]: attrs.source,\n }),\n timestamp,\n );\n};\n\nexport const recordApiRetry = (attrs: ApiRetryMetricAttrs) =>\n Metric.update(\n Metric.withAttributes(apiRetries, {\n [Attr.connectorName]: attrs.connector,\n [Attr.apiRetryReason]: attrs.reason,\n }),\n 1,\n );\n\nconst syncStates = [\"pending\", \"backfilling\", \"live\", \"error\"] as const;\n\nexport const setSyncState = (attrs: SyncStateAttrs, current: SyncState) =>\n Effect.forEach(\n syncStates,\n (state) =>\n Metric.update(\n Metric.withAttributes(syncState, {\n ...syncAttributes(attrs),\n [Attr.syncState]: state,\n }),\n state === current ? 1 : 0,\n ),\n { discard: true },\n );\n\n/** @internal Converts the ergonomic SDK input to canonical OTel attributes. */\nexport const withResourceAttributes = resourceAttributes;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AA6CA,MAAa,mBAAmB,OAAO,QAAQ,oCAAoC;CACjF,aAAa;CACb,aAAa;CACb,YAAY,EAAE,MAAM,cAAc;CACnC,CAAC;AAEF,MAAa,kBAAkB,OAAO,QAAQ,oCAAoC;CAChF,aAAa;CACb,aAAa;CACb,YAAY,EAAE,MAAM,cAAc;CACnC,CAAC;AAEF,MAAa,UAAU,OAAO,QAAQ,6BAA6B;CACjE,aAAa;CACb,aAAa;CACb,YAAY,EAAE,MAAM,WAAW;CAChC,CAAC;AAEF,MAAa,YAAY,OAAO,UAAU,gCAAgC;CACxE,aAAa;CACb,YAAY;EAAC;EAAG;EAAG;EAAI;EAAI;EAAI;EAAK;EAAK;EAAK;EAAK;CACnD,YAAY,EAAE,MAAM,cAAc;CACnC,CAAC;AAEF,MAAa,kBAAkB,OAAO,MAAM,sCAAsC;CAChF,aAAa;CACb,YAAY;EAAC;EAAG;EAAG;EAAI;EAAI;EAAI;EAAK;EAAK;EAAK;EAAM;EAAM;EAAM;EAAM;CACtE,YAAY,EAAE,MAAM,MAAM;CAC3B,CAAC;AAEF,MAAa,kBAAkB,OAAO,QAAQ,sCAAsC;CAClF,aAAa;CACb,aAAa;CACb,YAAY,EAAE,MAAM,aAAa;CAClC,CAAC;AAEF,MAAa,oBAAoB,OAAO,MAAM,yCAAyC;CACrF,aAAa;CACb,YAAY,EAAE,MAAM,WAAW;CAChC,CAAC;AAEF,MAAa,YAAY,OAAO,MAAM,gCAAgC;CACpE,aAAa;CACb,YAAY,EAAE,MAAM,KAAK;CAC1B,CAAC;AAEF,MAAa,uBAAuB,OAAO,MAAM,4CAA4C;CAC3F,aAAa;CACb,YAAY,EAAE,MAAM,KAAK;CAC1B,CAAC;AAEF,MAAa,aAAa,OAAO,QAAQ,iCAAiC;CACxE,aAAa;CACb,aAAa;CACb,YAAY,EAAE,MAAM,WAAW;CAChC,CAAC;AAEF,MAAM,sBAAsB,WAAgC;EACzD,KAAK,gBAAgB,MAAM;EAC3B,KAAK,eAAe,MAAM;EAC1B,KAAK,iBAAiB,MAAM;CAC9B;AAED,MAAM,kBAAkB,WAA2B;EAChD,KAAK,gBAAgB,MAAM;EAC3B,KAAK,eAAe,MAAM;CAC5B;AAED,MAAa,wBAAwB,UACnC,OAAO,OACL,OAAO,eAAe,iBAAiB;EACpC,KAAK,gBAAgB,MAAM;EAC3B,KAAK,cAAc,MAAM;EACzB,KAAK,iBAAiB,MAAM;CAC9B,CAAC,EACF,EACD;AAEH,MAAa,wBAAwB,WAAmB,UACtD,OAAO,OACL,OAAO,eAAe,mBAAmB,GAAG,KAAK,gBAAgB,WAAW,CAAC,EAC7E,MACD;AAEH,MAAa,2BAA2B,OAA+B,kBAA2B;CAChG,MAAM,YACJ,kBAAkB,SACd,IACA,SAAS,KAAK,cAAc,CAAC,KAC3B,OAAO,KAAK,aAAa,SAAS,cAAc,SAAS,GAAG,IAAK,EACjE,OAAO,gBAAgB,EAAE,CAC1B;AACP,QAAO,OAAO,OACZ,OAAO,eAAe,sBAAsB;GACzC,KAAK,gBAAgB,MAAM;GAC3B,KAAK,eAAe,MAAM;GAC1B,KAAK,iBAAiB,MAAM;EAC9B,CAAC,EACF,UACD;;AAGH,MAAa,kBAAkB,UAC7B,OAAO,OACL,OAAO,eAAe,YAAY;EAC/B,KAAK,gBAAgB,MAAM;EAC3B,KAAK,iBAAiB,MAAM;CAC9B,CAAC,EACF,EACD;AAEH,MAAM,aAAa;CAAC;CAAW;CAAe;CAAQ;CAAQ;AAE9D,MAAa,gBAAgB,OAAuB,YAClD,OAAO,QACL,aACC,UACC,OAAO,OACL,OAAO,eAAe,WAAW;CAC/B,GAAG,eAAe,MAAM;EACvB,KAAK,YAAY;CACnB,CAAC,EACF,UAAU,UAAU,IAAI,EACzB,EACH,EAAE,SAAS,MAAM,CAClB;;AAGH,MAAa,yBAAyB"}
|
package/dist/metrics.d.ts
DELETED
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import "./index-DKoRwyFn.js";
|
|
2
|
-
import "./types-jVD5Ys43.js";
|
|
3
|
-
import { C as syncState, E as withResourceAttributes, S as setWebhookQueueDepth, T as webhookRequests, _ as publishDuration, a as LastSuccessMetricAttrs, b as setLastSuccessTimestamp, c as WebhookMetricAttrs, d as batchSize, f as batches, h as lastSuccessTimestamp, i as BatchSource, l as WebhookOutcome, m as entitiesUpserted, n as ApiRetryReason, o as ResourceMetricAttrs, p as entitiesDeleted, r as BatchOutcome, s as SyncStateAttrs, t as ApiRetryMetricAttrs, u as apiRetries, v as recordApiRetry, w as webhookQueueDepth, x as setSyncState, y as recordWebhookRequest } from "./metrics-C4V1pm6v.js";
|
|
4
|
-
export { ApiRetryMetricAttrs, ApiRetryReason, BatchOutcome, BatchSource, LastSuccessMetricAttrs, ResourceMetricAttrs, SyncStateAttrs, WebhookMetricAttrs, WebhookOutcome, apiRetries, batchSize, batches, entitiesDeleted, entitiesUpserted, lastSuccessTimestamp, publishDuration, recordApiRetry, recordWebhookRequest, setLastSuccessTimestamp, setSyncState, setWebhookQueueDepth, syncState, webhookQueueDepth, webhookRequests, withResourceAttributes };
|
package/dist/metrics.js
DELETED
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import "./constants-D113Ut1x.js";
|
|
2
|
-
import "./telemetry-j_nxwYfh.js";
|
|
3
|
-
import { _ as withResourceAttributes, a as entitiesUpserted, c as publishDuration, d as setLastSuccessTimestamp, f as setSyncState, g as webhookRequests, h as webhookQueueDepth, i as entitiesDeleted, l as recordApiRetry, m as syncState, n as batchSize, o as lastSuccessTimestamp, p as setWebhookQueueDepth, r as batches, t as apiRetries, u as recordWebhookRequest } from "./metrics-inkCAxKq.js";
|
|
4
|
-
|
|
5
|
-
export { apiRetries, batchSize, batches, entitiesDeleted, entitiesUpserted, lastSuccessTimestamp, publishDuration, recordApiRetry, recordWebhookRequest, setLastSuccessTimestamp, setSyncState, setWebhookQueueDepth, syncState, webhookQueueDepth, webhookRequests, withResourceAttributes };
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import "../index-DKoRwyFn.js";
|
|
2
|
-
import { c as ResourceBatch } from "../types-jVD5Ys43.js";
|
|
3
|
-
import { a as PublisherService, i as Publisher, n as PublishOptions, r as PublishSource, t as PublishAck } from "../service-CUjGhe-7.js";
|
|
4
|
-
import "../index-ZSuiZX5H.js";
|
|
5
|
-
import { a as layerWingsConfig, i as layerWings, n as layerConsole, r as WingsPublisherConfig } from "../index-BA8ASHKr.js";
|
|
6
|
-
export { PublishAck, PublishOptions, PublishSource, Publisher, PublisherService, ResourceBatch, WingsPublisherConfig, layerConsole, layerWings, layerWingsConfig };
|
package/dist/publisher/index.js
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import "../errors-DHWylqRZ.js";
|
|
2
|
-
import "../constants-D113Ut1x.js";
|
|
3
|
-
import "../telemetry-j_nxwYfh.js";
|
|
4
|
-
import { t as Publisher } from "../service-DiP-XETo.js";
|
|
5
|
-
import { i as layerConsole, n as layerWings, r as layerWingsConfig } from "../publisher-BdWJOfer.js";
|
|
6
|
-
import "../table-bindings-Dp1DhTeS.js";
|
|
7
|
-
|
|
8
|
-
export { Publisher, layerConsole, layerWings, layerWingsConfig };
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
-
import { t as ConnectorError } from "./errors-DHWylqRZ.js";
|
|
3
|
-
import { i as SpanName } from "./telemetry-j_nxwYfh.js";
|
|
4
|
-
import { t as Publisher } from "./service-DiP-XETo.js";
|
|
5
|
-
import { t as loadTableBindings } from "./table-bindings-Dp1DhTeS.js";
|
|
6
|
-
import { Effect, Layer } from "effect";
|
|
7
|
-
import * as Wings from "@useairfoil/wings";
|
|
8
|
-
|
|
9
|
-
//#region src/publisher/console.ts
|
|
10
|
-
const layerConsole = Layer.succeed(Publisher)({ publish: ({ resource, source, batch }) => Effect.gen(function* () {
|
|
11
|
-
yield* Effect.logInfo(`[publisher] -> Source: ${source} | Resource: ${resource}`).pipe(Effect.annotateLogs({
|
|
12
|
-
mutations: batch.mutations.length,
|
|
13
|
-
cursor: batch.cursor,
|
|
14
|
-
source
|
|
15
|
-
}));
|
|
16
|
-
return {
|
|
17
|
-
status: "accepted",
|
|
18
|
-
resource
|
|
19
|
-
};
|
|
20
|
-
}) });
|
|
21
|
-
|
|
22
|
-
//#endregion
|
|
23
|
-
//#region src/publisher/wings.ts
|
|
24
|
-
const normalizeMapping = (mapping) => typeof mapping === "string" ? { name: mapping } : mapping;
|
|
25
|
-
const buildRecordBatch = (rows) => {
|
|
26
|
-
const [batch] = Wings.Arrow.tableFromJSON(Array.from(rows)).batches;
|
|
27
|
-
return batch ? Effect.succeed(batch) : Effect.fail(new ConnectorError({ message: "No rows to publish" }));
|
|
28
|
-
};
|
|
29
|
-
const validateTableMapping = (options) => Effect.try({
|
|
30
|
-
try: () => {
|
|
31
|
-
const keyField = Wings.TableUtils.getKeyField(options.table);
|
|
32
|
-
const versionField = Wings.TableUtils.getVersionField(options.table);
|
|
33
|
-
if (keyField.name !== options.resource.key) throw new Error(`Resource ${options.resource.name} key '${options.resource.key}' does not match Wings table key '${keyField.name}'`);
|
|
34
|
-
if (versionField.name !== options.resource.version) throw new Error(`Resource ${options.resource.name} version '${options.resource.version}' does not match Wings table version '${versionField.name}'`);
|
|
35
|
-
if (options.resource.partition?.required === true && !options.partitionValue) throw new Error(`Resource ${options.resource.name} requires a partition value`);
|
|
36
|
-
Wings.TableUtils.validatePartitionValueUnsafe(options.table, options.partitionValue);
|
|
37
|
-
return {
|
|
38
|
-
keyField,
|
|
39
|
-
versionField
|
|
40
|
-
};
|
|
41
|
-
},
|
|
42
|
-
catch: (cause) => new ConnectorError({
|
|
43
|
-
message: `Invalid Wings table mapping for ${options.resource.name}`,
|
|
44
|
-
cause
|
|
45
|
-
})
|
|
46
|
-
});
|
|
47
|
-
const accepted = (entry) => ({
|
|
48
|
-
status: "accepted",
|
|
49
|
-
resource: entry.resource.name,
|
|
50
|
-
partition: entry.partitionValue
|
|
51
|
-
});
|
|
52
|
-
const rejected = (entry, reason, rejectedRows) => ({
|
|
53
|
-
status: "rejected",
|
|
54
|
-
resource: entry.resource.name,
|
|
55
|
-
reason,
|
|
56
|
-
rejectedRows,
|
|
57
|
-
partition: entry.partitionValue
|
|
58
|
-
});
|
|
59
|
-
/** Builds a Wings publisher from explicit table mappings. */
|
|
60
|
-
const layerWings = (config) => Layer.effect(Publisher)(Effect.gen(function* () {
|
|
61
|
-
const entries = /* @__PURE__ */ new Map();
|
|
62
|
-
const clusterClient = yield* Wings.WingsClient.clusterClient;
|
|
63
|
-
for (const resource of config.connector.resources) {
|
|
64
|
-
const mappingInput = config.tables[resource.name];
|
|
65
|
-
if (!mappingInput) return yield* Effect.fail(new ConnectorError({ message: `Missing table for ${resource.name}` }));
|
|
66
|
-
const mapping = normalizeMapping(mappingInput);
|
|
67
|
-
const table = yield* clusterClient.getTable({ name: mapping.name }).pipe(Effect.mapError((error) => new ConnectorError({
|
|
68
|
-
message: `Failed to resolve Wings table for ${resource.name}`,
|
|
69
|
-
cause: error
|
|
70
|
-
})));
|
|
71
|
-
const { keyField, versionField } = yield* validateTableMapping({
|
|
72
|
-
resource,
|
|
73
|
-
table,
|
|
74
|
-
partitionValue: mapping.partitionValue
|
|
75
|
-
});
|
|
76
|
-
const publisher = yield* Wings.WingsClient.publisher({
|
|
77
|
-
table,
|
|
78
|
-
partitionValue: mapping.partitionValue
|
|
79
|
-
}).pipe(Effect.mapError((error) => new ConnectorError({
|
|
80
|
-
message: error.message,
|
|
81
|
-
cause: error
|
|
82
|
-
})));
|
|
83
|
-
entries.set(resource.name, {
|
|
84
|
-
resource,
|
|
85
|
-
table,
|
|
86
|
-
publisher,
|
|
87
|
-
partitionValue: mapping.partitionValue,
|
|
88
|
-
keyField,
|
|
89
|
-
versionField
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
const pushRows = (entry, operation, rows) => Effect.gen(function* () {
|
|
93
|
-
if (rows.length === 0) return accepted(entry);
|
|
94
|
-
const batch = yield* buildRecordBatch(rows);
|
|
95
|
-
return yield* entry.publisher.push({
|
|
96
|
-
operation,
|
|
97
|
-
batch,
|
|
98
|
-
partitionValue: entry.partitionValue
|
|
99
|
-
}).pipe(Effect.matchEffect({
|
|
100
|
-
onFailure: (error) => Effect.succeed(rejected(entry, error.message, rows.length)),
|
|
101
|
-
onSuccess: () => Effect.succeed(accepted(entry))
|
|
102
|
-
}));
|
|
103
|
-
});
|
|
104
|
-
const service = { publish: Effect.fn(SpanName.publish, { kind: "producer" })(function* ({ resource, source: _source, batch }) {
|
|
105
|
-
const entry = entries.get(resource);
|
|
106
|
-
if (!entry) return yield* Effect.fail(new ConnectorError({ message: `Unknown resource ${resource}` }));
|
|
107
|
-
if (batch.mutations.length === 0) return accepted(entry);
|
|
108
|
-
const upserts = [];
|
|
109
|
-
const deletes = [];
|
|
110
|
-
for (const mutation of batch.mutations) if (mutation.op === "upsert") upserts.push(mutation.row);
|
|
111
|
-
else deletes.push({
|
|
112
|
-
[entry.keyField.name]: mutation.key,
|
|
113
|
-
[entry.versionField.name]: mutation.version
|
|
114
|
-
});
|
|
115
|
-
const upsertAck = yield* pushRows(entry, "upsert", upserts);
|
|
116
|
-
if (upsertAck.status === "rejected") return upsertAck;
|
|
117
|
-
return yield* pushRows(entry, "delete", deletes);
|
|
118
|
-
}) };
|
|
119
|
-
return Publisher.of(service);
|
|
120
|
-
}));
|
|
121
|
-
/** Builds a Wings publisher from the complete hosted `AIRFOIL_TABLE_BINDINGS` document. */
|
|
122
|
-
const layerWingsConfig = (connector) => Layer.unwrap(loadTableBindings(connector).pipe(Effect.map((tables) => layerWings({
|
|
123
|
-
connector,
|
|
124
|
-
tables
|
|
125
|
-
}))));
|
|
126
|
-
|
|
127
|
-
//#endregion
|
|
128
|
-
//#region src/publisher/index.ts
|
|
129
|
-
var publisher_exports = /* @__PURE__ */ __export({
|
|
130
|
-
Publisher: () => Publisher,
|
|
131
|
-
layerConsole: () => layerConsole,
|
|
132
|
-
layerWings: () => layerWings,
|
|
133
|
-
layerWingsConfig: () => layerWingsConfig
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
//#endregion
|
|
137
|
-
export { layerConsole as i, layerWings as n, layerWingsConfig as r, publisher_exports as t };
|
|
138
|
-
//# sourceMappingURL=publisher-BdWJOfer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"publisher-BdWJOfer.js","names":["service: PublisherService","upserts: Rows[]","deletes: Rows[]"],"sources":["../src/publisher/console.ts","../src/publisher/wings.ts","../src/publisher/index.ts"],"sourcesContent":["import { Effect, Layer } from \"effect\";\n\nimport { Publisher } from \"./service\";\n\nexport const layerConsole = Layer.succeed(Publisher)({\n publish: ({ resource, source, batch }) =>\n Effect.gen(function* () {\n yield* Effect.logInfo(`[publisher] -> Source: ${source} | Resource: ${resource}`).pipe(\n Effect.annotateLogs({\n mutations: batch.mutations.length,\n cursor: batch.cursor,\n source,\n }),\n );\n\n return { status: \"accepted\" as const, resource };\n }),\n});\n","import * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition } from \"../core/types\";\n\nimport { ConnectorError } from \"../errors\";\nimport { loadTableBindings } from \"../runtime-config/table-bindings\";\nimport { SpanName } from \"../telemetry\";\nimport { Publisher, type PublishAck, type PublisherService } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\n/** Programmatic mapping between a connector resource and a Wings table. */\nexport type WingsTableMapping = {\n readonly name: string;\n readonly partitionValue?: Wings.PartitionValue.PartitionValue;\n};\n\n/** Direct Wings publisher configuration, mainly for tests and local runtimes. */\nexport type WingsPublisherConfig = {\n readonly connector: ConnectorDefinition;\n readonly tables: Record<string, string | WingsTableMapping>;\n};\n\ntype PublisherEntry = {\n readonly resource: ResourceDefinition;\n readonly table: Wings.Cluster.Table.Table;\n readonly publisher: Wings.WingsClient.Publisher;\n readonly partitionValue?: Wings.PartitionValue.PartitionValue;\n readonly keyField: Wings.Cluster.ArrowType.Field;\n readonly versionField: Wings.Cluster.ArrowType.Field;\n};\n\nconst normalizeMapping = (mapping: string | WingsTableMapping): WingsTableMapping =>\n typeof mapping === \"string\" ? { name: mapping } : mapping;\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n const table = Wings.Arrow.tableFromJSON(Array.from(rows));\n const [batch] = table.batches;\n return batch\n ? Effect.succeed(batch)\n : Effect.fail(new ConnectorError({ message: \"No rows to publish\" }));\n};\n\nconst validateTableMapping = (options: {\n readonly resource: ResourceDefinition;\n readonly table: Wings.Cluster.Table.Table;\n readonly partitionValue?: Wings.PartitionValue.PartitionValue;\n}) =>\n Effect.try({\n try: () => {\n const keyField = Wings.TableUtils.getKeyField(options.table);\n const versionField = Wings.TableUtils.getVersionField(options.table);\n\n if (keyField.name !== options.resource.key) {\n throw new Error(\n `Resource ${options.resource.name} key '${options.resource.key}' does not match Wings table key '${keyField.name}'`,\n );\n }\n if (versionField.name !== options.resource.version) {\n throw new Error(\n `Resource ${options.resource.name} version '${options.resource.version}' does not match Wings table version '${versionField.name}'`,\n );\n }\n if (options.resource.partition?.required === true && !options.partitionValue) {\n throw new Error(`Resource ${options.resource.name} requires a partition value`);\n }\n Wings.TableUtils.validatePartitionValueUnsafe(options.table, options.partitionValue);\n\n return { keyField, versionField };\n },\n catch: (cause) =>\n new ConnectorError({\n message: `Invalid Wings table mapping for ${options.resource.name}`,\n cause,\n }),\n });\n\nconst accepted = (entry: PublisherEntry): PublishAck => ({\n status: \"accepted\",\n resource: entry.resource.name,\n partition: entry.partitionValue,\n});\n\nconst rejected = (entry: PublisherEntry, reason: string, rejectedRows: number): PublishAck => ({\n status: \"rejected\",\n resource: entry.resource.name,\n reason,\n rejectedRows,\n partition: entry.partitionValue,\n});\n\n/** Builds a Wings publisher from explicit table mappings. */\nexport const layerWings = (\n config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n Layer.effect(Publisher)(\n Effect.gen(function* () {\n const entries = new Map<string, PublisherEntry>();\n const clusterClient = yield* Wings.WingsClient.clusterClient;\n\n for (const resource of config.connector.resources) {\n const mappingInput = config.tables[resource.name];\n if (!mappingInput) {\n return yield* Effect.fail(\n new ConnectorError({ message: `Missing table for ${resource.name}` }),\n );\n }\n\n const mapping = normalizeMapping(mappingInput);\n const table = yield* clusterClient.getTable({ name: mapping.name }).pipe(\n Effect.mapError(\n (error) =>\n new ConnectorError({\n message: `Failed to resolve Wings table for ${resource.name}`,\n cause: error,\n }),\n ),\n );\n const { keyField, versionField } = yield* validateTableMapping({\n resource,\n table,\n partitionValue: mapping.partitionValue,\n });\n\n const publisher = yield* Wings.WingsClient.publisher({\n table,\n partitionValue: mapping.partitionValue,\n }).pipe(\n Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),\n );\n\n entries.set(resource.name, {\n resource,\n table,\n publisher,\n partitionValue: mapping.partitionValue,\n keyField,\n versionField,\n });\n }\n\n const pushRows = (\n entry: PublisherEntry,\n operation: \"upsert\" | \"delete\",\n rows: ReadonlyArray<Rows>,\n ) =>\n Effect.gen(function* () {\n if (rows.length === 0) return accepted(entry);\n const batch = yield* buildRecordBatch(rows);\n return yield* entry.publisher\n .push({ operation, batch, partitionValue: entry.partitionValue })\n .pipe(\n Effect.matchEffect({\n onFailure: (error) => Effect.succeed(rejected(entry, error.message, rows.length)),\n onSuccess: () => Effect.succeed(accepted(entry)),\n }),\n );\n });\n\n const service: PublisherService = {\n publish: Effect.fn(SpanName.publish, { kind: \"producer\" })(function* ({\n resource,\n source: _source,\n batch,\n }) {\n const entry = entries.get(resource);\n if (!entry) {\n return yield* Effect.fail(\n new ConnectorError({ message: `Unknown resource ${resource}` }),\n );\n }\n\n if (batch.mutations.length === 0) return accepted(entry);\n\n const upserts: Rows[] = [];\n const deletes: Rows[] = [];\n for (const mutation of batch.mutations) {\n if (mutation.op === \"upsert\") {\n upserts.push(mutation.row as Rows);\n } else {\n deletes.push({\n [entry.keyField.name]: mutation.key,\n [entry.versionField.name]: mutation.version,\n });\n }\n }\n\n const upsertAck = yield* pushRows(entry, \"upsert\", upserts);\n if (upsertAck.status === \"rejected\") return upsertAck;\n\n return yield* pushRows(entry, \"delete\", deletes);\n }),\n };\n\n return Publisher.of(service);\n }),\n );\n\n/** Builds a Wings publisher from the complete hosted `AIRFOIL_TABLE_BINDINGS` document. */\nexport const layerWingsConfig = (connector: ConnectorDefinition) =>\n Layer.unwrap(\n loadTableBindings(connector).pipe(Effect.map((tables) => layerWings({ connector, tables }))),\n );\n","export type { ResourceBatch } from \"../core/types\";\nexport type { PublishAck, PublishOptions, PublisherService, PublishSource } from \"./service\";\nexport type { WingsPublisherConfig } from \"./wings\";\n\nexport { layerConsole } from \"./console\";\nexport { Publisher } from \"./service\";\nexport { layerWings, layerWingsConfig } from \"./wings\";\n"],"mappings":";;;;;;;;;AAIA,MAAa,eAAe,MAAM,QAAQ,UAAU,CAAC,EACnD,UAAU,EAAE,UAAU,QAAQ,YAC5B,OAAO,IAAI,aAAa;AACtB,QAAO,OAAO,QAAQ,0BAA0B,OAAO,eAAe,WAAW,CAAC,KAChF,OAAO,aAAa;EAClB,WAAW,MAAM,UAAU;EAC3B,QAAQ,MAAM;EACd;EACD,CAAC,CACH;AAED,QAAO;EAAE,QAAQ;EAAqB;EAAU;EAChD,EACL,CAAC;;;;ACgBF,MAAM,oBAAoB,YACxB,OAAO,YAAY,WAAW,EAAE,MAAM,SAAS,GAAG;AAEpD,MAAM,oBAAoB,SAA8B;CAEtD,MAAM,CAAC,SADO,MAAM,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CACnC;AACtB,QAAO,QACH,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC,CAAC;;AAGxE,MAAM,wBAAwB,YAK5B,OAAO,IAAI;CACT,WAAW;EACT,MAAM,WAAW,MAAM,WAAW,YAAY,QAAQ,MAAM;EAC5D,MAAM,eAAe,MAAM,WAAW,gBAAgB,QAAQ,MAAM;AAEpE,MAAI,SAAS,SAAS,QAAQ,SAAS,IACrC,OAAM,IAAI,MACR,YAAY,QAAQ,SAAS,KAAK,QAAQ,QAAQ,SAAS,IAAI,oCAAoC,SAAS,KAAK,GAClH;AAEH,MAAI,aAAa,SAAS,QAAQ,SAAS,QACzC,OAAM,IAAI,MACR,YAAY,QAAQ,SAAS,KAAK,YAAY,QAAQ,SAAS,QAAQ,wCAAwC,aAAa,KAAK,GAClI;AAEH,MAAI,QAAQ,SAAS,WAAW,aAAa,QAAQ,CAAC,QAAQ,eAC5D,OAAM,IAAI,MAAM,YAAY,QAAQ,SAAS,KAAK,6BAA6B;AAEjF,QAAM,WAAW,6BAA6B,QAAQ,OAAO,QAAQ,eAAe;AAEpF,SAAO;GAAE;GAAU;GAAc;;CAEnC,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,mCAAmC,QAAQ,SAAS;EAC7D;EACD,CAAC;CACL,CAAC;AAEJ,MAAM,YAAY,WAAuC;CACvD,QAAQ;CACR,UAAU,MAAM,SAAS;CACzB,WAAW,MAAM;CAClB;AAED,MAAM,YAAY,OAAuB,QAAgB,kBAAsC;CAC7F,QAAQ;CACR,UAAU,MAAM,SAAS;CACzB;CACA;CACA,WAAW,MAAM;CAClB;;AAGD,MAAa,cACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;CACjD,MAAM,gBAAgB,OAAO,MAAM,YAAY;AAE/C,MAAK,MAAM,YAAY,OAAO,UAAU,WAAW;EACjD,MAAM,eAAe,OAAO,OAAO,SAAS;AAC5C,MAAI,CAAC,aACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,SAAS,QAAQ,CAAC,CACtE;EAGH,MAAM,UAAU,iBAAiB,aAAa;EAC9C,MAAM,QAAQ,OAAO,cAAc,SAAS,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,KAClE,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,qCAAqC,SAAS;GACvD,OAAO;GACR,CAAC,CACL,CACF;EACD,MAAM,EAAE,UAAU,iBAAiB,OAAO,qBAAqB;GAC7D;GACA;GACA,gBAAgB,QAAQ;GACzB,CAAC;EAEF,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,QAAQ;GACzB,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAAC,CACzF;AAED,UAAQ,IAAI,SAAS,MAAM;GACzB;GACA;GACA;GACA,gBAAgB,QAAQ;GACxB;GACA;GACD,CAAC;;CAGJ,MAAM,YACJ,OACA,WACA,SAEA,OAAO,IAAI,aAAa;AACtB,MAAI,KAAK,WAAW,EAAG,QAAO,SAAS,MAAM;EAC7C,MAAM,QAAQ,OAAO,iBAAiB,KAAK;AAC3C,SAAO,OAAO,MAAM,UACjB,KAAK;GAAE;GAAW;GAAO,gBAAgB,MAAM;GAAgB,CAAC,CAChE,KACC,OAAO,YAAY;GACjB,YAAY,UAAU,OAAO,QAAQ,SAAS,OAAO,MAAM,SAAS,KAAK,OAAO,CAAC;GACjF,iBAAiB,OAAO,QAAQ,SAAS,MAAM,CAAC;GACjD,CAAC,CACH;GACH;CAEJ,MAAMA,UAA4B,EAChC,SAAS,OAAO,GAAG,SAAS,SAAS,EAAE,MAAM,YAAY,CAAC,CAAC,WAAW,EACpE,UACA,QAAQ,SACR,SACC;EACD,MAAM,QAAQ,QAAQ,IAAI,SAAS;AACnC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,oBAAoB,YAAY,CAAC,CAChE;AAGH,MAAI,MAAM,UAAU,WAAW,EAAG,QAAO,SAAS,MAAM;EAExD,MAAMC,UAAkB,EAAE;EAC1B,MAAMC,UAAkB,EAAE;AAC1B,OAAK,MAAM,YAAY,MAAM,UAC3B,KAAI,SAAS,OAAO,SAClB,SAAQ,KAAK,SAAS,IAAY;MAElC,SAAQ,KAAK;IACV,MAAM,SAAS,OAAO,SAAS;IAC/B,MAAM,aAAa,OAAO,SAAS;GACrC,CAAC;EAIN,MAAM,YAAY,OAAO,SAAS,OAAO,UAAU,QAAQ;AAC3D,MAAI,UAAU,WAAW,WAAY,QAAO;AAE5C,SAAO,OAAO,SAAS,OAAO,UAAU,QAAQ;GAChD,EACH;AAED,QAAO,UAAU,GAAG,QAAQ;EAC5B,CACH;;AAGH,MAAa,oBAAoB,cAC/B,MAAM,OACJ,kBAAkB,UAAU,CAAC,KAAK,OAAO,KAAK,WAAW,WAAW;CAAE;CAAW;CAAQ,CAAC,CAAC,CAAC,CAC7F"}
|
|
@@ -1,4 +0,0 @@
|
|
|
1
|
-
import "../index-DKoRwyFn.js";
|
|
2
|
-
import "../types-jVD5Ys43.js";
|
|
3
|
-
import { a as loadTableBindings, c as PlatformRuntimeKey, d as RuntimeConfigError, f as RuntimeConfigErrorCode, i as ResolvedTableBindings, l as httpPort, n as HostedTableBinding, o as layerHosted, r as ResolvedTableBinding, s as PlatformRuntimeDefault, u as wingsClient } from "../index-ZSuiZX5H.js";
|
|
4
|
-
export { HostedTableBinding, PlatformRuntimeDefault, PlatformRuntimeKey, ResolvedTableBinding, ResolvedTableBindings, RuntimeConfigError, RuntimeConfigErrorCode, httpPort, layerHosted, loadTableBindings, wingsClient };
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import { n as PlatformRuntimeKey, t as PlatformRuntimeDefault } from "../constants-D113Ut1x.js";
|
|
2
|
-
import { n as RuntimeConfigError, t as loadTableBindings } from "../table-bindings-Dp1DhTeS.js";
|
|
3
|
-
import { i as wingsClient, n as layerHosted, r as httpPort } from "../runtime-config-IbyCVpei.js";
|
|
4
|
-
|
|
5
|
-
export { PlatformRuntimeDefault, PlatformRuntimeKey, RuntimeConfigError, httpPort, layerHosted, loadTableBindings, wingsClient };
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
-
import { n as PlatformRuntimeKey, t as PlatformRuntimeDefault } from "./constants-D113Ut1x.js";
|
|
3
|
-
import { n as RuntimeConfigError, t as loadTableBindings } from "./table-bindings-Dp1DhTeS.js";
|
|
4
|
-
import { Config, ConfigProvider, Effect, FileSystem, Schema } from "effect";
|
|
5
|
-
|
|
6
|
-
//#region src/runtime-config/config.ts
|
|
7
|
-
/** Shared HTTP port used by hosted and sandbox connector entrypoints. */
|
|
8
|
-
const httpPort = Config.port(PlatformRuntimeKey.httpPort).pipe(Config.withDefault(PlatformRuntimeDefault.httpPort));
|
|
9
|
-
/** Shared Wings client configuration supplied by the hosting platform. */
|
|
10
|
-
const wingsClient = {
|
|
11
|
-
host: Config.nonEmptyString(PlatformRuntimeKey.wingsHost),
|
|
12
|
-
namespace: Config.nonEmptyString(PlatformRuntimeKey.wingsNamespace)
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
//#endregion
|
|
16
|
-
//#region src/runtime-config/layer.ts
|
|
17
|
-
const providerFromFile = Effect.gen(function* () {
|
|
18
|
-
const configPath = yield* Config.nonEmptyString(PlatformRuntimeKey.configPath).pipe(Effect.mapError((error) => new RuntimeConfigError({
|
|
19
|
-
code: "CONFIG_PATH_MISSING",
|
|
20
|
-
message: `${PlatformRuntimeKey.configPath} is required in hosted mode`,
|
|
21
|
-
cause: error.cause
|
|
22
|
-
})));
|
|
23
|
-
const contents = yield* (yield* FileSystem.FileSystem).readFileString(configPath).pipe(Effect.mapError((cause) => new RuntimeConfigError({
|
|
24
|
-
code: "CONFIG_FILE_UNREADABLE",
|
|
25
|
-
message: "Connector config file could not be read",
|
|
26
|
-
cause
|
|
27
|
-
})));
|
|
28
|
-
const document = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(contents).pipe(Effect.mapError((error) => new RuntimeConfigError({
|
|
29
|
-
code: "CONFIG_JSON_INVALID",
|
|
30
|
-
message: "Connector config is not valid JSON",
|
|
31
|
-
cause: error
|
|
32
|
-
})));
|
|
33
|
-
return ConfigProvider.fromUnknown(document, { preserveEmptyStrings: true });
|
|
34
|
-
});
|
|
35
|
-
/**
|
|
36
|
-
* Loads the required JSON file and lets environment values override it per key.
|
|
37
|
-
* Connector field validation remains in the connector's Effect Config.
|
|
38
|
-
*
|
|
39
|
-
* @example
|
|
40
|
-
* ```ts
|
|
41
|
-
* program.pipe(Effect.provide(layerHosted()));
|
|
42
|
-
* ```
|
|
43
|
-
*/
|
|
44
|
-
const layerHosted = () => ConfigProvider.layerAdd(providerFromFile);
|
|
45
|
-
|
|
46
|
-
//#endregion
|
|
47
|
-
//#region src/runtime-config/index.ts
|
|
48
|
-
var runtime_config_exports = /* @__PURE__ */ __export({
|
|
49
|
-
PlatformRuntimeDefault: () => PlatformRuntimeDefault,
|
|
50
|
-
PlatformRuntimeKey: () => PlatformRuntimeKey,
|
|
51
|
-
RuntimeConfigError: () => RuntimeConfigError,
|
|
52
|
-
httpPort: () => httpPort,
|
|
53
|
-
layerHosted: () => layerHosted,
|
|
54
|
-
loadTableBindings: () => loadTableBindings,
|
|
55
|
-
wingsClient: () => wingsClient
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
//#endregion
|
|
59
|
-
export { wingsClient as i, layerHosted as n, httpPort as r, runtime_config_exports as t };
|
|
60
|
-
//# sourceMappingURL=runtime-config-IbyCVpei.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-config-IbyCVpei.js","names":["providerFromFile: Effect.Effect<\n ConfigProvider.ConfigProvider,\n RuntimeConfigError,\n FileSystem.FileSystem\n>"],"sources":["../src/runtime-config/config.ts","../src/runtime-config/layer.ts","../src/runtime-config/index.ts"],"sourcesContent":["import type { WingsClientOptions } from \"@useairfoil/wings/wings-client\";\n\nimport { Config } from \"effect\";\n\nimport { PlatformRuntimeDefault, PlatformRuntimeKey } from \"./constants\";\n\n/** Shared HTTP port used by hosted and sandbox connector entrypoints. */\nexport const httpPort = Config.port(PlatformRuntimeKey.httpPort).pipe(\n Config.withDefault(PlatformRuntimeDefault.httpPort),\n);\n\n/** Shared Wings client configuration supplied by the hosting platform. */\nexport const wingsClient = {\n host: Config.nonEmptyString(PlatformRuntimeKey.wingsHost),\n namespace: Config.nonEmptyString(PlatformRuntimeKey.wingsNamespace),\n} satisfies Config.Wrap<WingsClientOptions>;\n","import { Config, ConfigProvider, Effect, FileSystem, Schema } from \"effect\";\n\nimport { PlatformRuntimeKey } from \"./constants\";\nimport { RuntimeConfigError } from \"./error\";\n\nconst providerFromFile: Effect.Effect<\n ConfigProvider.ConfigProvider,\n RuntimeConfigError,\n FileSystem.FileSystem\n> = Effect.gen(function* () {\n const configPath = yield* Config.nonEmptyString(PlatformRuntimeKey.configPath).pipe(\n Effect.mapError(\n (error) =>\n new RuntimeConfigError({\n code: \"CONFIG_PATH_MISSING\",\n message: `${PlatformRuntimeKey.configPath} is required in hosted mode`,\n cause: error.cause,\n }),\n ),\n );\n const fileSystem = yield* FileSystem.FileSystem;\n const contents = yield* fileSystem.readFileString(configPath).pipe(\n Effect.mapError(\n (cause) =>\n new RuntimeConfigError({\n code: \"CONFIG_FILE_UNREADABLE\",\n message: \"Connector config file could not be read\",\n cause,\n }),\n ),\n );\n const document = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(\n contents,\n ).pipe(\n Effect.mapError(\n (error) =>\n new RuntimeConfigError({\n code: \"CONFIG_JSON_INVALID\",\n message: \"Connector config is not valid JSON\",\n cause: error,\n }),\n ),\n );\n return ConfigProvider.fromUnknown(document, { preserveEmptyStrings: true });\n});\n\n/**\n * Loads the required JSON file and lets environment values override it per key.\n * Connector field validation remains in the connector's Effect Config.\n *\n * @example\n * ```ts\n * program.pipe(Effect.provide(layerHosted()));\n * ```\n */\nexport const layerHosted = () => ConfigProvider.layerAdd(providerFromFile);\n","export { RuntimeConfigError, type RuntimeConfigErrorCode } from \"./error\";\nexport { httpPort, wingsClient } from \"./config\";\nexport { PlatformRuntimeDefault, PlatformRuntimeKey } from \"./constants\";\nexport { layerHosted } from \"./layer\";\nexport {\n type HostedTableBinding,\n type ResolvedTableBinding,\n type ResolvedTableBindings,\n loadTableBindings,\n} from \"./table-bindings\";\n"],"mappings":";;;;;;;AAOA,MAAa,WAAW,OAAO,KAAK,mBAAmB,SAAS,CAAC,KAC/D,OAAO,YAAY,uBAAuB,SAAS,CACpD;;AAGD,MAAa,cAAc;CACzB,MAAM,OAAO,eAAe,mBAAmB,UAAU;CACzD,WAAW,OAAO,eAAe,mBAAmB,eAAe;CACpE;;;;ACVD,MAAMA,mBAIF,OAAO,IAAI,aAAa;CAC1B,MAAM,aAAa,OAAO,OAAO,eAAe,mBAAmB,WAAW,CAAC,KAC7E,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS,GAAG,mBAAmB,WAAW;EAC1C,OAAO,MAAM;EACd,CAAC,CACL,CACF;CAED,MAAM,WAAW,QADE,OAAO,WAAW,YACF,eAAe,WAAW,CAAC,KAC5D,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS;EACT;EACD,CAAC,CACL,CACF;CACD,MAAM,WAAW,OAAO,OAAO,oBAAoB,OAAO,eAAe,OAAO,QAAQ,CAAC,CACvF,SACD,CAAC,KACA,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS;EACT,OAAO;EACR,CAAC,CACL,CACF;AACD,QAAO,eAAe,YAAY,UAAU,EAAE,sBAAsB,MAAM,CAAC;EAC3E;;;;;;;;;;AAWF,MAAa,oBAAoB,eAAe,SAAS,iBAAiB"}
|
package/dist/server-D-eLX3yN.js
DELETED
|
@@ -1,183 +0,0 @@
|
|
|
1
|
-
import { t as ConnectorError } from "./errors-DHWylqRZ.js";
|
|
2
|
-
import { a as addCurrentSpanEvent, i as SpanName, n as EventAttr, o as annotateError, r as EventName, t as Attr } from "./telemetry-j_nxwYfh.js";
|
|
3
|
-
import { _ as withResourceAttributes, a as entitiesUpserted, c as publishDuration, i as entitiesDeleted, n as batchSize, p as setWebhookQueueDepth, r as batches, u as recordWebhookRequest } from "./metrics-inkCAxKq.js";
|
|
4
|
-
import { t as Publisher } from "./service-DiP-XETo.js";
|
|
5
|
-
import { Cause, Context, Effect, Metric, Queue, Schema } from "effect";
|
|
6
|
-
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
|
|
7
|
-
|
|
8
|
-
//#region src/publisher/instrumented.ts
|
|
9
|
-
const publishBatch = Effect.fnUntraced(function* (options) {
|
|
10
|
-
const metric = {
|
|
11
|
-
connector: options.connector ?? "unknown",
|
|
12
|
-
resource: options.resource,
|
|
13
|
-
source: options.source
|
|
14
|
-
};
|
|
15
|
-
const metricAttributes = withResourceAttributes(metric);
|
|
16
|
-
const upserts = options.batch.mutations.filter((mutation) => mutation.op === "upsert").length;
|
|
17
|
-
const deletes = options.batch.mutations.length - upserts;
|
|
18
|
-
const ack = yield* (yield* Publisher).publish({
|
|
19
|
-
resource: options.resource,
|
|
20
|
-
source: options.source,
|
|
21
|
-
batch: options.batch
|
|
22
|
-
}).pipe(Effect.trackDuration(Metric.withAttributes(publishDuration, metricAttributes)), Effect.catchCause((cause) => Metric.update(Metric.withAttributes(batches, {
|
|
23
|
-
...metricAttributes,
|
|
24
|
-
[Attr.batchOutcome]: "error"
|
|
25
|
-
}), 1).pipe(Effect.andThen(Effect.failCause(cause)))));
|
|
26
|
-
yield* Metric.update(Metric.withAttributes(batches, {
|
|
27
|
-
...metricAttributes,
|
|
28
|
-
[Attr.batchOutcome]: ack.status
|
|
29
|
-
}), 1);
|
|
30
|
-
yield* Effect.annotateCurrentSpan({ [Attr.publisherSuccess]: ack.status === "accepted" });
|
|
31
|
-
if (ack.status === "rejected") {
|
|
32
|
-
yield* Effect.annotateCurrentSpan({ [Attr.errorPhase]: "publish" });
|
|
33
|
-
return yield* Effect.fail(new ConnectorError({ message: `Publisher rejected batch for ${options.resource}: ${ack.reason}` }));
|
|
34
|
-
}
|
|
35
|
-
yield* Metric.update(Metric.withAttributes(entitiesUpserted, metricAttributes), upserts);
|
|
36
|
-
yield* Metric.update(Metric.withAttributes(entitiesDeleted, metricAttributes), deletes);
|
|
37
|
-
yield* Metric.update(Metric.withAttributes(batchSize, metricAttributes), options.batch.mutations.length);
|
|
38
|
-
if (options.batch.cursor !== void 0) yield* addCurrentSpanEvent(EventName.batchCheckpoint, { [EventAttr.batchCursor]: options.batch.cursor });
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
//#endregion
|
|
42
|
-
//#region src/webhook/server.ts
|
|
43
|
-
var WebhookQueue = class extends Context.Service()("@useairfoil/connector-kit/WebhookQueue") {};
|
|
44
|
-
const resourceBatch = (mutations) => ({ mutations });
|
|
45
|
-
const decodeAndHandleWebhook = (resourceName, webhook, payload) => Schema.decodeUnknownEffect(webhook.schema)(payload).pipe(Effect.mapError((cause) => new ConnectorError({
|
|
46
|
-
message: `Invalid webhook payload for resource ${resourceName}`,
|
|
47
|
-
cause
|
|
48
|
-
})), Effect.flatMap((decoded) => webhook.handler({ payload: decoded })));
|
|
49
|
-
const makeRouteContext = (request, rawBody, payload, batches$1) => ({
|
|
50
|
-
request,
|
|
51
|
-
rawBody,
|
|
52
|
-
payload,
|
|
53
|
-
to: (resource, payload$1) => Effect.gen(function* () {
|
|
54
|
-
if (!resource.webhook) return yield* Effect.fail(new ConnectorError({ message: `Resource ${resource.name} does not define a webhook handler` }));
|
|
55
|
-
const mutations = yield* decodeAndHandleWebhook(resource.name, resource.webhook, payload$1);
|
|
56
|
-
batches$1.push({
|
|
57
|
-
resource: resource.name,
|
|
58
|
-
batch: resourceBatch(mutations)
|
|
59
|
-
});
|
|
60
|
-
})
|
|
61
|
-
});
|
|
62
|
-
const compactBatches = (batches$1) => {
|
|
63
|
-
const grouped = /* @__PURE__ */ new Map();
|
|
64
|
-
for (const item of batches$1) {
|
|
65
|
-
const current = grouped.get(item.resource) ?? [];
|
|
66
|
-
grouped.set(item.resource, [...current, ...item.batch.mutations]);
|
|
67
|
-
}
|
|
68
|
-
return Array.from(grouped.entries()).map(([resource, mutations]) => ({
|
|
69
|
-
resource,
|
|
70
|
-
batch: resourceBatch(mutations)
|
|
71
|
-
}));
|
|
72
|
-
};
|
|
73
|
-
const publishBatches = (batches$1, options) => Effect.gen(function* () {
|
|
74
|
-
yield* Effect.forEach(batches$1, (batch) => publishBatch({
|
|
75
|
-
connector: options.connectorName,
|
|
76
|
-
resource: batch.resource,
|
|
77
|
-
source: "webhook",
|
|
78
|
-
batch: batch.batch
|
|
79
|
-
}), { concurrency: "unbounded" });
|
|
80
|
-
});
|
|
81
|
-
const badRequest = (message) => HttpServerResponse.jsonUnsafe({
|
|
82
|
-
ok: false,
|
|
83
|
-
error: message
|
|
84
|
-
}, { status: 400 });
|
|
85
|
-
const readRawBody = (request) => request.arrayBuffer.pipe(Effect.map((buffer) => new Uint8Array(buffer)));
|
|
86
|
-
const parseJson = (rawBody) => Effect.try({
|
|
87
|
-
try: () => JSON.parse(new TextDecoder().decode(rawBody)),
|
|
88
|
-
catch: (cause) => new ConnectorError({
|
|
89
|
-
message: "Invalid webhook JSON",
|
|
90
|
-
cause
|
|
91
|
-
})
|
|
92
|
-
});
|
|
93
|
-
const makeHandler = (route, options) => Effect.gen(function* () {
|
|
94
|
-
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
95
|
-
const collected = [];
|
|
96
|
-
const rawBodyResult = yield* readRawBody(request).pipe(Effect.match({
|
|
97
|
-
onFailure: () => ({
|
|
98
|
-
_tag: "Error",
|
|
99
|
-
message: "Failed to read webhook body"
|
|
100
|
-
}),
|
|
101
|
-
onSuccess: (value) => ({
|
|
102
|
-
_tag: "Success",
|
|
103
|
-
value
|
|
104
|
-
})
|
|
105
|
-
}));
|
|
106
|
-
if (rawBodyResult._tag === "Error") {
|
|
107
|
-
yield* recordWebhookRequest({
|
|
108
|
-
connector: options.connectorName,
|
|
109
|
-
path: String(route.path),
|
|
110
|
-
outcome: "read_error"
|
|
111
|
-
});
|
|
112
|
-
return badRequest(rawBodyResult.message);
|
|
113
|
-
}
|
|
114
|
-
const jsonResult = yield* parseJson(rawBodyResult.value).pipe(Effect.match({
|
|
115
|
-
onFailure: (error) => ({
|
|
116
|
-
_tag: "Error",
|
|
117
|
-
message: error.message
|
|
118
|
-
}),
|
|
119
|
-
onSuccess: (value) => ({
|
|
120
|
-
_tag: "Success",
|
|
121
|
-
value
|
|
122
|
-
})
|
|
123
|
-
}));
|
|
124
|
-
if (jsonResult._tag === "Error") {
|
|
125
|
-
yield* recordWebhookRequest({
|
|
126
|
-
connector: options.connectorName,
|
|
127
|
-
path: String(route.path),
|
|
128
|
-
outcome: "invalid_json"
|
|
129
|
-
});
|
|
130
|
-
return badRequest(jsonResult.message);
|
|
131
|
-
}
|
|
132
|
-
const payloadResult = yield* Schema.decodeUnknownEffect(route.schema)(jsonResult.value).pipe(Effect.match({
|
|
133
|
-
onFailure: () => ({
|
|
134
|
-
_tag: "Error",
|
|
135
|
-
message: "Invalid webhook payload"
|
|
136
|
-
}),
|
|
137
|
-
onSuccess: (value) => ({
|
|
138
|
-
_tag: "Success",
|
|
139
|
-
value
|
|
140
|
-
})
|
|
141
|
-
}));
|
|
142
|
-
if (payloadResult._tag === "Error") {
|
|
143
|
-
yield* recordWebhookRequest({
|
|
144
|
-
connector: options.connectorName,
|
|
145
|
-
path: String(route.path),
|
|
146
|
-
outcome: "invalid_payload"
|
|
147
|
-
});
|
|
148
|
-
return badRequest(payloadResult.message);
|
|
149
|
-
}
|
|
150
|
-
const response = yield* Effect.withSpan(route.handler(makeRouteContext(request, rawBodyResult.value, payloadResult.value, collected)).pipe(Effect.tapError((error) => annotateError("webhook_handle", error))), SpanName.webhookHandle, { attributes: { [Attr.webhookPath]: route.path } });
|
|
151
|
-
const batches$1 = compactBatches(collected);
|
|
152
|
-
if (route.ackMode === "after-enqueue") {
|
|
153
|
-
const queue = yield* WebhookQueue;
|
|
154
|
-
yield* Queue.offerAll(queue.queue, batches$1);
|
|
155
|
-
const depth = yield* Queue.size(queue.queue);
|
|
156
|
-
yield* setWebhookQueueDepth(options.connectorName, depth);
|
|
157
|
-
yield* recordWebhookRequest({
|
|
158
|
-
connector: options.connectorName,
|
|
159
|
-
path: String(route.path),
|
|
160
|
-
outcome: response.status >= 400 ? "rejected" : "ok"
|
|
161
|
-
});
|
|
162
|
-
return response;
|
|
163
|
-
}
|
|
164
|
-
yield* Effect.withSpan(publishBatches(batches$1, { connectorName: options.connectorName }).pipe(Effect.tapError((error) => annotateError("webhook_publish", error))), SpanName.webhookHandle, { attributes: { [Attr.webhookPath]: route.path } });
|
|
165
|
-
yield* recordWebhookRequest({
|
|
166
|
-
connector: options.connectorName,
|
|
167
|
-
path: String(route.path),
|
|
168
|
-
outcome: response.status >= 400 ? "rejected" : "ok"
|
|
169
|
-
});
|
|
170
|
-
return response;
|
|
171
|
-
}).pipe(Effect.catchCause((cause) => recordWebhookRequest({
|
|
172
|
-
connector: options.connectorName,
|
|
173
|
-
path: String(route.path),
|
|
174
|
-
outcome: "handler_error"
|
|
175
|
-
}).pipe(Effect.andThen(Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`)), Effect.andThen(Effect.succeed(HttpServerResponse.jsonUnsafe({
|
|
176
|
-
ok: false,
|
|
177
|
-
error: "Webhook handler failed"
|
|
178
|
-
}, { status: 500 }))))));
|
|
179
|
-
const router = (routes, options) => HttpRouter.addAll(routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route, options))));
|
|
180
|
-
|
|
181
|
-
//#endregion
|
|
182
|
-
export { router as n, publishBatch as r, WebhookQueue as t };
|
|
183
|
-
//# sourceMappingURL=server-D-eLX3yN.js.map
|