@useairfoil/connector-kit 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +279 -90
  2. package/dist/chunk-Bp6m_JJh.js +13 -0
  3. package/dist/connector-app/index.d.ts +9 -0
  4. package/dist/connector-app/index.js +8 -0
  5. package/dist/connector-app-CvqbfN6R.js +32 -0
  6. package/dist/connector-app-CvqbfN6R.js.map +1 -0
  7. package/dist/errors/index.d.ts +2 -0
  8. package/dist/errors/index.js +3 -0
  9. package/dist/errors-DHWylqRZ.js +8 -0
  10. package/dist/errors-DHWylqRZ.js.map +1 -0
  11. package/dist/index-8KnVpoZp.d.ts +32 -0
  12. package/dist/index-BlhJCpEk.d.ts +124 -0
  13. package/dist/index-CQJsFJs3.d.ts +23 -0
  14. package/dist/index-DK9YcR6W.d.ts +25 -0
  15. package/dist/index-DaNW63Mv.d.ts +21 -0
  16. package/dist/index-DyuT2jzP.d.ts +14 -0
  17. package/dist/index.d.ts +9 -166
  18. package/dist/index.js +44 -229
  19. package/dist/index.js.map +1 -1
  20. package/dist/ingestion/index.d.ts +6 -0
  21. package/dist/ingestion/index.js +7 -0
  22. package/dist/ingestion-CMCroK7Y.js +171 -0
  23. package/dist/ingestion-CMCroK7Y.js.map +1 -0
  24. package/dist/publisher/index.d.ts +5 -0
  25. package/dist/publisher/index.js +5 -0
  26. package/dist/publisher-DusgYvaV.js +131 -0
  27. package/dist/publisher-DusgYvaV.js.map +1 -0
  28. package/dist/server-DAeFWT4o.js +113 -0
  29. package/dist/server-DAeFWT4o.js.map +1 -0
  30. package/dist/service-B7g_lHMI.d.ts +32 -0
  31. package/dist/state-store/index.d.ts +4 -0
  32. package/dist/state-store/index.js +3 -0
  33. package/dist/state-store-DG_4QpWG.js +31 -0
  34. package/dist/state-store-DG_4QpWG.js.map +1 -0
  35. package/dist/telemetry-p0dE5Mdu.js +171 -0
  36. package/dist/telemetry-p0dE5Mdu.js.map +1 -0
  37. package/dist/types-ZNrA-Dn0.d.ts +160 -0
  38. package/dist/webhook/index.d.ts +5 -0
  39. package/dist/webhook/index.js +6 -0
  40. package/dist/webhook-BSA-tXtU.js +13 -0
  41. package/dist/webhook-BSA-tXtU.js.map +1 -0
  42. package/package.json +41 -14
  43. package/src/connector-app/index.ts +1 -0
  44. package/src/connector-app/start.ts +38 -0
  45. package/src/core/builder.ts +73 -15
  46. package/src/core/index.ts +22 -0
  47. package/src/core/types.ts +186 -78
  48. package/src/errors/index.ts +1 -0
  49. package/src/index.ts +8 -28
  50. package/src/ingestion/engine.ts +302 -206
  51. package/src/ingestion/index.ts +1 -0
  52. package/src/publisher/console.ts +18 -0
  53. package/src/publisher/index.ts +7 -0
  54. package/src/publisher/service.ts +33 -14
  55. package/src/publisher/wings.ts +162 -82
  56. package/src/runtime/context.ts +15 -0
  57. package/src/state-store/index.ts +2 -0
  58. package/src/state-store/memory.ts +25 -0
  59. package/src/state-store/service.ts +18 -0
  60. package/src/telemetry.ts +246 -0
  61. package/src/webhook/index.ts +8 -0
  62. package/src/webhook/server.ts +177 -54
  63. package/src/webhook/types.ts +6 -16
  64. package/src/ingestion/state-store.ts +0 -36
  65. package/src/streams/pull-stream.ts +0 -55
  66. package/src/streams/webhook-queue.ts +0 -15
  67. /package/src/{core/errors.ts → errors/connector-error.ts} +0 -0
package/README.md CHANGED
@@ -1,127 +1,316 @@
1
1
  # @useairfoil/connector-kit
2
2
 
3
- Toolkit for building Airfoil connectors with Effect.
3
+ Toolkit for building Airfoil producer connectors with Effect.
4
4
 
5
- ---
5
+ `@useairfoil/connector-kit` gives connector authors the pieces needed to define, run, test, and publish resource-first producer connectors:
6
6
 
7
- ## User guide (build a connector)
7
+ - `Connector.define(...)` describes a connector.
8
+ - `Resource.entity(...)` describes a Wings table-backed resource.
9
+ - `Fetch.page(...)` and `Fetch.changes(...)` describe backfill and polling fetches.
10
+ - `Resource.webhook(...)` lets a resource normalize provider webhook payloads into mutations.
11
+ - `Webhook.route(...)` defines connector-level HTTP webhook routes.
12
+ - `Ingestion.run(...)` runs resource ingestion and checkpoints after accepted publishes.
13
+ - `Publisher.layerConsole` logs accepted batches locally.
14
+ - `Publisher.layerWings(...)` publishes resource mutations into Wings tables.
15
+ - `ConnectorApp.start(...)` starts a complete Node HTTP connector process.
8
16
 
9
- This section is for connector authors who want to build and run a connector.
17
+ The package is intentionally Layer-oriented: connector code defines resources and routes, while entrypoints provide runtime dependencies such as publishers, state stores, telemetry, and API clients.
10
18
 
11
- ### Install
19
+ ## Install
12
20
 
13
21
  ```bash
14
- bun add @useairfoil/connector-kit
22
+ pnpm add @useairfoil/connector-kit effect@beta
15
23
  ```
16
24
 
17
- ### Minimal example
25
+ This repo currently uses Effect v4 beta. In workspace packages, prefer the workspace catalog (`"effect": "catalog:"`) so all packages stay on the pinned beta version.
26
+
27
+ ## Package Shape
28
+
29
+ The package root exposes the common connector authoring API:
18
30
 
19
31
  ```ts
20
- import { Schema, Effect, Layer, Stream } from "effect";
21
32
  import {
22
- defineConnector,
23
- defineEntity,
33
+ Connector,
34
+ ConnectorApp,
35
+ ConnectorError,
36
+ Cursor,
37
+ Fetch,
38
+ Ingestion,
24
39
  Publisher,
25
- runConnector,
26
- StateStoreInMemory,
27
- WebhookServerLayer,
28
- makeWebhookQueue,
40
+ Resource,
41
+ StateStore,
42
+ Telemetry,
43
+ Webhook,
29
44
  } from "@useairfoil/connector-kit";
45
+ ```
46
+
47
+ Subpath exports are available for runtime domains:
48
+
49
+ - `@useairfoil/connector-kit/ingestion`
50
+ - `@useairfoil/connector-kit/connector-app`
51
+ - `@useairfoil/connector-kit/publisher`
52
+ - `@useairfoil/connector-kit/state-store`
53
+ - `@useairfoil/connector-kit/webhook`
54
+ - `@useairfoil/connector-kit/errors`
30
55
 
31
- const Customer = Schema.Struct({
32
- id: Schema.String,
33
- email: Schema.String,
34
- created_at: Schema.Date,
56
+ Core definition helpers and `Telemetry` are intentionally root-only.
57
+
58
+ ## Resource Model
59
+
60
+ Resources emit mutations, not raw rows. Connector Kit currently supports `Resource.entity(...)` for Wings table-backed resources.
61
+
62
+ ```ts
63
+ const Posts = Resource.entity({
64
+ name: "posts",
65
+ schema: PostSchema,
66
+ key: "id",
67
+ version: "updatedAt",
68
+ backfill: Fetch.page({
69
+ pageCursor: Cursor.number(),
70
+ cutoff: Cursor.isoDateTime(),
71
+ fetch: ({ pageCursor }) =>
72
+ api.fetchPosts({ page: typeof pageCursor === "number" ? pageCursor : 1 }).pipe(
73
+ Effect.map((page) => ({
74
+ mutations: page.items.map(Resource.upsert),
75
+ nextPageCursor: page.hasMore ? page.nextPage : page.page,
76
+ hasMore: page.hasMore,
77
+ })),
78
+ ),
79
+ }),
35
80
  });
81
+ ```
36
82
 
37
- const program = Effect.gen(function* () {
38
- const webhook = yield* makeWebhookQueue<Schema.Schema.Type<typeof Customer>>();
39
-
40
- const connector = defineConnector({
41
- name: "producer-example",
42
- entities: [
43
- defineEntity({
44
- name: "customers",
45
- schema: Customer,
46
- primaryKey: "id",
47
- live: webhook.queue,
48
- backfill: Stream.empty,
49
- }),
50
- ],
51
- events: [],
52
- });
53
-
54
- yield* runConnector(connector, new Date());
55
- }).pipe(
56
- Effect.provide(StateStoreInMemory),
57
- Effect.provide(
58
- Layer.succeed(Publisher, {
59
- publish: () => Effect.succeed({ requestId: 1n }),
60
- }),
61
- ),
62
- Effect.provide(
63
- WebhookServerLayer({
64
- port: 8080,
65
- routes: [
66
- {
67
- path: "/webhook/customers",
68
- schema: Customer,
69
- dispatch: (payload) =>
70
- Effect.succeed([
71
- {
72
- queue: webhook.queue,
73
- batch: { cursor: new Date(), rows: [payload] },
74
- },
75
- ]),
76
- },
77
- ],
83
+ `key` and `version` autocomplete from the decoded schema row type. Invalid field names are rejected at compile time.
84
+
85
+ Mutations are created with:
86
+
87
+ ```ts
88
+ Resource.upsert(row);
89
+
90
+ Resource.delete({
91
+ key: "post_123",
92
+ version: "2026-01-01T00:00:00.000Z",
93
+ });
94
+ ```
95
+
96
+ ## Fetches
97
+
98
+ `Fetch.page(...)` is for paginated backfill. It receives the stored page cursor and the resource cutoff.
99
+
100
+ `Fetch.changes(...)` is for polling incremental changes. It receives the stored changes cursor and can be configured with an interval.
101
+
102
+ Cursor helpers:
103
+
104
+ - `Cursor.string()`
105
+ - `Cursor.number()`
106
+ - `Cursor.isoDateTime()`
107
+ - `Cursor.nowIsoDateTime`
108
+
109
+ Date cursor values are accepted at API boundaries and normalized to ISO strings before checkpoint writes.
110
+
111
+ ## Webhooks
112
+
113
+ Webhooks are connector-level routes with schema-validated payloads. A route dispatches to resource-owned webhook handlers by calling `to(resource, payload)`.
114
+
115
+ ```ts
116
+ const Posts = Resource.entity({
117
+ name: "posts",
118
+ schema: PostSchema,
119
+ key: "id",
120
+ version: "updatedAt",
121
+ webhook: Resource.webhook({
122
+ schema: PostEventSchema,
123
+ handler: ({ payload }) => Effect.succeed([Resource.upsert(payload.data)]),
124
+ }),
125
+ });
126
+
127
+ const route = Webhook.route({
128
+ path: "/webhooks/example",
129
+ ackMode: "after-publish",
130
+ schema: ProviderWebhookSchema,
131
+ handler: ({ request, rawBody, payload, to }) =>
132
+ Effect.gen(function* () {
133
+ yield* verifySignature({ request, rawBody });
134
+ yield* to(Posts, payload);
135
+ return HttpServerResponse.jsonUnsafe({ ok: true });
78
136
  }),
79
- ),
80
- );
137
+ });
138
+ ```
139
+
140
+ Route behavior:
141
+
142
+ - reads the raw body once
143
+ - parses JSON once
144
+ - validates the route payload schema
145
+ - passes `request`, `rawBody`, typed `payload`, and `to(...)` to the handler
146
+ - returns `400` for invalid body reads, invalid JSON, or invalid route payloads
147
+ - returns `500` for unexpected handler/runtime/publisher failures
148
+
149
+ Signature verification is connector-owned and should use `rawBody`.
150
+
151
+ `ackMode: "after-publish"` publishes collected mutations before the response is completed. `ackMode: "after-enqueue"` enqueues collected mutations and lets the background webhook consumer publish them.
152
+
153
+ ## Connector Definition
81
154
 
82
- Effect.runPromise(program);
155
+ ```ts
156
+ const connector = Connector.define({
157
+ name: "producer-example",
158
+ title: "Producer Example",
159
+ resources: [Posts],
160
+ webhooks: [route],
161
+ });
83
162
  ```
84
163
 
85
- ---
164
+ ## Ingestion
165
+
166
+ `Ingestion.run(...)` contains the runtime engine. `ConnectorApp.start(...)` is the usual entrypoint for runnable producer CLIs.
86
167
 
87
- ## Development (concepts and architecture)
168
+ Current runtime behavior:
88
169
 
89
- ### Core concepts
170
+ - runs resource backfill, changes, and webhooks concurrently
171
+ - initializes the same initial cutoff for backfill and changes
172
+ - publishes through `Publisher.Publisher`
173
+ - checkpoints only after an accepted publish ACK
174
+ - does not checkpoint rejected publishes
175
+ - allows empty accepted batches to advance state
176
+ - persists resource state through `StateStore.StateStore`
90
177
 
91
- - `defineConnector` describes your connector and its entities.
92
- - `defineEntity` wires live and backfill streams for each entity.
93
- - `Publisher` is the output boundary (where batches go).
94
- - `StateStore` tracks cursors and backfill state.
95
- - `WebhookServerLayer` turns webhook routes into an HTTP server.
178
+ Use `StateStore.layerMemory` for development and tests, or provide your own durable `StateStore` implementation.
96
179
 
97
- ### Layers and Effect services
180
+ ## Publisher
98
181
 
99
- Connector-kit is designed around Effect services and Layers. Your application should provide:
182
+ `Publisher.Publisher` is the Effect service boundary for publishing resource mutation batches.
100
183
 
101
- - a `Publisher` Layer
102
- - a `StateStore` Layer
103
- - an HTTP server Layer (if you use webhooks)
104
- - any custom services your connector needs (API clients, Effect Config)
184
+ `Publisher.layerConsole` accepts batches and logs them locally.
105
185
 
106
- ### Testing with VCR
186
+ `Publisher.layerWings(...)` publishes mutations to Wings tables:
187
+
188
+ ```ts
189
+ const publisherLayer = Publisher.layerWings({
190
+ connector,
191
+ tables: {
192
+ posts: "namespaces/default/tables/posts",
193
+ },
194
+ });
195
+ ```
107
196
 
108
- VCR is provided via `@useairfoil/effect-http-client` as an Effect `HttpClient` layer. This keeps HTTP recording out of connector logic.
197
+ The `tables` map is keyed by resource name. Values can be table names or objects with a table name and optional partition value.
109
198
 
110
199
  ```ts
111
- import { FetchHttpClient } from "@effect/platform";
112
- import { NodeFileSystem } from "@effect/platform-node";
113
- import { CassetteStoreLive, VcrHttpClientLayer } from "@useairfoil/effect-http-client";
114
- import { Layer } from "effect";
200
+ Publisher.layerWings({
201
+ connector,
202
+ tables: {
203
+ posts: {
204
+ name: "namespaces/default/tables/posts",
205
+ partitionValue: "account_123",
206
+ },
207
+ },
208
+ });
209
+ ```
210
+
211
+ The Wings publisher resolves table metadata during layer construction, validates resource key/version/partition compatibility, sends upserts with full rows, and sends deletes with key/version-only rows.
212
+
213
+ ## Minimal Example
214
+
215
+ ```ts
216
+ import { NodeRuntime } from "@effect/platform-node";
217
+ import {
218
+ Connector,
219
+ ConnectorApp,
220
+ Cursor,
221
+ Fetch,
222
+ Publisher,
223
+ Resource,
224
+ StateStore,
225
+ Telemetry,
226
+ } from "@useairfoil/connector-kit";
227
+ import { ConfigProvider, Effect, Layer, Schema } from "effect";
228
+ import { FetchHttpClient } from "effect/unstable/http";
229
+
230
+ const PostSchema = Schema.Struct({
231
+ id: Schema.Number,
232
+ title: Schema.String,
233
+ updatedAt: Schema.String,
234
+ });
235
+
236
+ type Post = Schema.Schema.Type<typeof PostSchema>;
115
237
 
116
- const cassetteLayer = CassetteStoreLive.pipe(
117
- Layer.provide(NodeFileSystem.layer),
238
+ const Posts = Resource.entity({
239
+ name: "posts",
240
+ schema: PostSchema,
241
+ key: "id",
242
+ version: "updatedAt",
243
+ backfill: Fetch.page({
244
+ pageCursor: Cursor.number(),
245
+ cutoff: Cursor.isoDateTime(),
246
+ fetch: () =>
247
+ Effect.succeed({
248
+ mutations: [
249
+ Resource.upsert({ id: 1, title: "Hello", updatedAt: "2026-01-01T00:00:00.000Z" }),
250
+ ],
251
+ nextPageCursor: 1,
252
+ hasMore: false,
253
+ }),
254
+ }),
255
+ });
256
+
257
+ const connector = Connector.define({
258
+ name: "producer-example",
259
+ resources: [Posts],
260
+ });
261
+
262
+ const EnvLayer = Layer.mergeAll(
263
+ FetchHttpClient.layer,
264
+ Layer.succeed(ConfigProvider.ConfigProvider, ConfigProvider.fromEnv()),
118
265
  );
119
266
 
120
- const vcrLayer = VcrHttpClientLayer({
121
- cassetteDir: "cassettes",
122
- cassetteName: "example",
123
- mode: "auto",
124
- }).pipe(
125
- Layer.provide(Layer.mergeAll(FetchHttpClient.layer, cassetteLayer)),
267
+ const TelemetryLayer = Telemetry.layerOtlpTracing().pipe(Layer.provide(EnvLayer));
268
+ const runtimeLayer = Layer.mergeAll(StateStore.layerMemory, Publisher.layerConsole, TelemetryLayer);
269
+
270
+ Effect.scoped(ConnectorApp.start(connector, { port: 8080 })).pipe(
271
+ Effect.provide(runtimeLayer),
272
+ NodeRuntime.runMain,
126
273
  );
127
274
  ```
275
+
276
+ ## Telemetry
277
+
278
+ `Telemetry` contains connector-kit span names, span attributes, error annotation helpers, and OTLP tracing layers.
279
+
280
+ Common entry points:
281
+
282
+ - `Telemetry.layer(config, options?)`
283
+ - `Telemetry.layerConfig(config, options?)`
284
+ - `Telemetry.layerOtlpTracing(options?)`
285
+
286
+ Default redacted headers:
287
+
288
+ - `authorization`
289
+ - `cookie`
290
+ - `set-cookie`
291
+ - `x-api-key`
292
+ - `/api[-_]?key/i`
293
+ - `/secret/i`
294
+ - `/signature/i`
295
+ - `/token/i`
296
+
297
+ Telemetry environment variables:
298
+
299
+ | Variable | Read by | Description |
300
+ | ----------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------- |
301
+ | `OTEL_ENABLED` | Connector Kit | Enables trace export when `true`. Defaults to `false`. |
302
+ | `OTEL_EXPORTER_OTLP_ENDPOINT` | Connector Kit | OTLP base URL, for example `http://localhost:4318`. Connector Kit appends `/v1/traces`. Required when `OTEL_ENABLED=true`. |
303
+ | `OTEL_EXPORTER_OTLP_HEADERS` | Connector Kit | Optional comma-separated headers, for example `Authorization=Bearer <token>,X-Axiom-Dataset=<dataset>`. |
304
+ | `OTEL_SERVICE_NAME` | Effect | Service name resource attribute, for example `producer-shopify`. |
305
+ | `OTEL_SERVICE_VERSION` | Effect | Optional service version resource attribute. |
306
+ | `OTEL_RESOURCE_ATTRIBUTES` | Effect | Optional comma-separated resource attributes, for example `deployment.environment=production,team=data`. |
307
+
308
+ ## Testing
309
+
310
+ For tests, the most common setup is:
311
+
312
+ - `StateStore.layerMemory` for state
313
+ - a small in-memory `Publisher.Publisher` test layer
314
+ - `Ingestion.run(...)` or `ConnectorApp.start(...)` inside `Effect.scoped`
315
+
316
+ Keep HTTP recording concerns outside connector logic by providing a VCR-backed `HttpClient` Layer.
@@ -0,0 +1,13 @@
1
+ //#region rolldown:runtime
2
+ var __defProp = Object.defineProperty;
3
+ var __export = (all) => {
4
+ let target = {};
5
+ for (var name in all) __defProp(target, name, {
6
+ get: all[name],
7
+ enumerable: true
8
+ });
9
+ return target;
10
+ };
11
+
12
+ //#endregion
13
+ export { __export as t };
@@ -0,0 +1,9 @@
1
+ import "../index-DyuT2jzP.js";
2
+ import "../types-ZNrA-Dn0.js";
3
+ import { i as start, n as App, r as StartOptions } from "../index-BlhJCpEk.js";
4
+ import "../service-B7g_lHMI.js";
5
+ import "../index-DaNW63Mv.js";
6
+ import "../index-8KnVpoZp.js";
7
+ import "../index-DK9YcR6W.js";
8
+ import "../index-CQJsFJs3.js";
9
+ export { App, StartOptions, start };
@@ -0,0 +1,8 @@
1
+ import "../errors-DHWylqRZ.js";
2
+ import "../telemetry-p0dE5Mdu.js";
3
+ import "../state-store-DG_4QpWG.js";
4
+ import "../server-DAeFWT4o.js";
5
+ import "../ingestion-CMCroK7Y.js";
6
+ import { n as start } from "../connector-app-CvqbfN6R.js";
7
+
8
+ export { start };
@@ -0,0 +1,32 @@
1
+ import { t as __export } from "./chunk-Bp6m_JJh.js";
2
+ import { n as run } from "./ingestion-CMCroK7Y.js";
3
+ import { DateTime, Effect } from "effect";
4
+ import { NodeHttpServer } from "@effect/platform-node";
5
+ import { HttpRouter } from "effect/unstable/http";
6
+ import { createServer } from "node:http";
7
+
8
+ //#region src/connector-app/start.ts
9
+ const start = (app, options) => Effect.gen(function* () {
10
+ const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
11
+ const routes = app.webhooks?.map((route) => route.path) ?? [];
12
+ yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
13
+ port: options.port,
14
+ routes
15
+ }));
16
+ return yield* run(app, {
17
+ initialCutoff,
18
+ webhook: {
19
+ routes: app.webhooks ?? [],
20
+ healthPath: options.healthPath ?? "/health",
21
+ disableHttpLogger: options.disableHttpLogger ?? true
22
+ }
23
+ }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
24
+ });
25
+
26
+ //#endregion
27
+ //#region src/connector-app/index.ts
28
+ var connector_app_exports = /* @__PURE__ */ __export({ start: () => start });
29
+
30
+ //#endregion
31
+ export { start as n, connector_app_exports as t };
32
+ //# sourceMappingURL=connector-app-CvqbfN6R.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connector-app-CvqbfN6R.js","names":["runIngestion"],"sources":["../src/connector-app/start.ts","../src/connector-app/index.ts"],"sourcesContent":["import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/unstable/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n readonly port: number;\n readonly initialCutoff?: Cursor.Value;\n readonly healthPath?: HttpRouter.PathInput;\n readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n Effect.gen(function* () {\n const initialCutoff =\n options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n yield* Effect.logInfo(\"webhook server ready\").pipe(\n Effect.annotateLogs({ port: options.port, routes }),\n );\n\n return yield* runIngestion(app, {\n initialCutoff,\n webhook: {\n routes: app.webhooks ?? [],\n healthPath: options.healthPath ?? \"/health\",\n disableHttpLogger: options.disableHttpLogger ?? true,\n },\n }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n });\n","export { start, type App, type StartOptions } from \"./start\";\n"],"mappings":";;;;;;;;AAmBA,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;CACpF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,KAAK,IAAI,EAAE;AAE7D,QAAO,OAAO,QAAQ,uBAAuB,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;EAAQ,CAAC,CACpD;AAED,QAAO,OAAOA,IAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,EAAE;GAC1B,YAAY,QAAQ,cAAc;GAClC,mBAAmB,QAAQ,qBAAqB;GACjD;EACF,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,CAAC;EACnF"}
@@ -0,0 +1,2 @@
1
+ import { t as ConnectorError } from "../index-DyuT2jzP.js";
2
+ export { ConnectorError };
@@ -0,0 +1,3 @@
1
+ import { t as ConnectorError } from "../errors-DHWylqRZ.js";
2
+
3
+ export { ConnectorError };
@@ -0,0 +1,8 @@
1
+ import { Data } from "effect";
2
+
3
+ //#region src/errors/connector-error.ts
4
+ var ConnectorError = class extends Data.TaggedError("ConnectorError") {};
5
+
6
+ //#endregion
7
+ export { ConnectorError as t };
8
+ //# sourceMappingURL=errors-DHWylqRZ.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors-DHWylqRZ.js","names":[],"sources":["../src/errors/connector-error.ts"],"sourcesContent":["import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n readonly message: string;\n readonly cause?: unknown;\n}> {}\n"],"mappings":";;;AAEA,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE"}
@@ -0,0 +1,32 @@
1
+ import { t as ConnectorError } from "./index-DyuT2jzP.js";
2
+ import { l as ResourceDefinition, n as ConnectorDefinition, r as Cursor, y as WebhookRoute } from "./types-ZNrA-Dn0.js";
3
+ import { i as Publisher } from "./service-B7g_lHMI.js";
4
+ import { r as StateStore } from "./index-DaNW63Mv.js";
5
+ import { Effect } from "effect";
6
+ import { HttpRouter, HttpServer } from "effect/unstable/http";
7
+
8
+ //#region src/ingestion/engine.d.ts
9
+ type RunBaseOptions = {
10
+ readonly initialCutoff?: Cursor.Value;
11
+ };
12
+ type RunOptions = RunBaseOptions & {
13
+ readonly webhook?: {
14
+ readonly routes: ReadonlyArray<WebhookRoute>;
15
+ readonly healthPath?: HttpRouter.PathInput;
16
+ readonly disableHttpLogger?: boolean;
17
+ };
18
+ };
19
+ type RunNoWebhookOptions = RunBaseOptions & {
20
+ readonly webhook?: undefined;
21
+ };
22
+ type RunWebhookOptions = RunOptions & {
23
+ readonly webhook: NonNullable<RunOptions["webhook"]>;
24
+ };
25
+ declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options?: RunNoWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Publisher>;
26
+ declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options: RunWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
27
+ declare namespace index_d_exports {
28
+ export { RunOptions, run };
29
+ }
30
+ //#endregion
31
+ export { RunOptions as n, run as r, index_d_exports as t };
32
+ //# sourceMappingURL=index-8KnVpoZp.d.ts.map
@@ -0,0 +1,124 @@
1
+ import { t as ConnectorError } from "./index-DyuT2jzP.js";
2
+ import { d as ResourceMutation, h as ResourceSchema, i as DeleteValue, l as ResourceDefinition, n as ConnectorDefinition, r as Cursor, s as PageFetch, t as ChangesFetch, v as WebhookHandler } from "./types-ZNrA-Dn0.js";
3
+ import { i as Publisher } from "./service-B7g_lHMI.js";
4
+ import { r as StateStore } from "./index-DaNW63Mv.js";
5
+ import { Config, Effect, Layer } from "effect";
6
+ import { HttpRouter } from "effect/unstable/http";
7
+ import * as effect_unstable_http_HttpClient0 from "effect/unstable/http/HttpClient";
8
+ import * as effect_unstable_http_HttpServerError0 from "effect/unstable/http/HttpServerError";
9
+
10
+ //#region src/core/builder.d.ts
11
+ declare const Connector: {
12
+ define: <const Resources extends ReadonlyArray<ResourceDefinition>>(definition: ConnectorDefinition<Resources>) => ConnectorDefinition<Resources>;
13
+ };
14
+ declare const Resource: {
15
+ entity: <S extends ResourceSchema, Payload = never, R = never>(definition: ResourceDefinition<S, Payload, R>) => ResourceDefinition<S, Payload, R>;
16
+ webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) => WebhookHandler<Row, Payload>;
17
+ upsert: <Row extends object>(row: Row) => ResourceMutation<Row>;
18
+ delete: (options: {
19
+ readonly key: DeleteValue;
20
+ readonly version: DeleteValue;
21
+ }) => ResourceMutation<never>;
22
+ };
23
+ declare const Fetch: {
24
+ page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => PageFetch<Row, R>;
25
+ changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => ChangesFetch<Row, R>;
26
+ };
27
+ declare const Cursor$1: {
28
+ string: () => Cursor.Definition<string>;
29
+ number: () => Cursor.Definition<number>;
30
+ isoDateTime: () => Cursor.Definition<string>;
31
+ nowIsoDateTime: Effect.Effect<string, never, never>;
32
+ };
33
+ declare namespace telemetry_d_exports {
34
+ export { Attr, EventAttr, EventName, OtlpTracingConfig, OtlpTracingOptions, SpanName, addCurrentSpanEvent, annotateError, layer, layerConfig, layerOtlpTracing };
35
+ }
36
+ /** Canonical span names emitted by connector-kit runtime instrumentation. */
37
+ declare const SpanName: {
38
+ readonly batchProcess: "connector.batch.process";
39
+ readonly webhookDecode: "connector.webhook.decode";
40
+ readonly webhookHandle: "connector.webhook.handle";
41
+ readonly publish: "connector.publish";
42
+ readonly stateSet: "connector.state.set";
43
+ readonly apiFetch: "connector.api.fetch";
44
+ };
45
+ /** Canonical span attribute names. Keep these stable for collector queries. */
46
+ declare const Attr: {
47
+ readonly connectorName: "airfoil.connector.name";
48
+ readonly connectorResourcesCount: "airfoil.connector.resources.count";
49
+ readonly resourceName: "airfoil.resource.name";
50
+ readonly resourceSource: "airfoil.resource.source";
51
+ readonly batchMutations: "airfoil.batch.mutations";
52
+ readonly webhookPath: "airfoil.webhook.path";
53
+ readonly publisherSuccess: "airfoil.publisher.success";
54
+ readonly stateKey: "airfoil.state.key";
55
+ readonly apiPath: "airfoil.api.path";
56
+ readonly errorPhase: "airfoil.error.phase";
57
+ readonly errorType: "airfoil.error.type";
58
+ readonly errorMessage: "airfoil.error.message";
59
+ readonly errorDetails: "airfoil.error.details";
60
+ };
61
+ /** Span event names for high-cardinality reproduction details. */
62
+ declare const EventName: {
63
+ readonly batchCheckpoint: "airfoil.batch.checkpoint";
64
+ };
65
+ /** Event-only attributes. These are intentionally not span attributes. */
66
+ declare const EventAttr: {
67
+ readonly batchCursor: "airfoil.batch.cursor";
68
+ };
69
+ /** Annotates the active span with phase-aware error metadata before re-failing. */
70
+ declare const annotateError: (phase: string, error: unknown) => Effect.Effect<void, never, never>;
71
+ /** Adds an event to the active span, or no-ops when no span is active. */
72
+ declare const addCurrentSpanEvent: (name: string, attributes?: Record<string, unknown>) => Effect.Effect<void, never, never>;
73
+ /** Transport configuration for OTLP trace export. All fields are direct values. */
74
+ type OtlpTracingConfig = {
75
+ readonly enabled?: boolean;
76
+ readonly endpoint?: string;
77
+ readonly headers?: Record<string, string>;
78
+ };
79
+ /** Runtime options — not config-wrappable since RegExp cannot live in Config. */
80
+ type OtlpTracingOptions = {
81
+ /** Additional connector-specific header names or regexes to redact from HTTP logs/traces. */
82
+ readonly redactedHeaders?: ReadonlyArray<string | RegExp>;
83
+ };
84
+ /**
85
+ * Trace-only OTLP export with sensitive HTTP header redaction.
86
+ * Takes direct config values — no `ConfigProvider` required.
87
+ */
88
+ declare const layer: (config?: OtlpTracingConfig, options?: OtlpTracingOptions) => Layer.Layer<never, Error, effect_unstable_http_HttpClient0.HttpClient>;
89
+ /**
90
+ * Trace-only OTLP export with sensitive HTTP header redaction.
91
+ * Takes Effect Config-wrapped values — reads from `ConfigProvider`.
92
+ *
93
+ * @example
94
+ * Telemetry.layerConfig({
95
+ * enabled: Config.boolean("MY_OTEL_ENABLED"),
96
+ * endpoint: Config.string("MY_COLLECTOR_URL"),
97
+ * })
98
+ */
99
+ declare const layerConfig: (config: Config.Wrap<OtlpTracingConfig>, options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
100
+ /**
101
+ * Trace-only OTLP export with sensitive HTTP header redaction.
102
+ * Zero-config shortcut that reads the standard `OTEL_*` env var names.
103
+ *
104
+ * Reads `OTEL_ENABLED`, `OTEL_EXPORTER_OTLP_ENDPOINT`, and
105
+ * `OTEL_EXPORTER_OTLP_HEADERS`. Effect's OTLP resource handling reads
106
+ * `OTEL_SERVICE_NAME`, `OTEL_SERVICE_VERSION`, and `OTEL_RESOURCE_ATTRIBUTES`.
107
+ */
108
+ declare const layerOtlpTracing: (options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
109
+ //#endregion
110
+ //#region src/connector-app/start.d.ts
111
+ type App = ConnectorDefinition;
112
+ type StartOptions = {
113
+ readonly port: number;
114
+ readonly initialCutoff?: Cursor.Value;
115
+ readonly healthPath?: HttpRouter.PathInput;
116
+ readonly disableHttpLogger?: boolean;
117
+ };
118
+ declare const start: (app: App, options: StartOptions) => Effect.Effect<void, ConnectorError | effect_unstable_http_HttpServerError0.ServeError, StateStore | Publisher>;
119
+ declare namespace index_d_exports {
120
+ export { App, StartOptions, start };
121
+ }
122
+ //#endregion
123
+ export { telemetry_d_exports as a, Fetch as c, start as i, Resource as l, App as n, Connector as o, StartOptions as r, Cursor$1 as s, index_d_exports as t };
124
+ //# sourceMappingURL=index-BlhJCpEk.d.ts.map