@useairfoil/connector-kit 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +283 -90
- package/dist/chunk-Bp6m_JJh.js +13 -0
- package/dist/connector-app/index.d.ts +9 -0
- package/dist/connector-app/index.js +8 -0
- package/dist/connector-app-CvqbfN6R.js +32 -0
- package/dist/connector-app-CvqbfN6R.js.map +1 -0
- package/dist/errors/index.d.ts +2 -0
- package/dist/errors/index.js +3 -0
- package/dist/errors-DHWylqRZ.js +8 -0
- package/dist/errors-DHWylqRZ.js.map +1 -0
- package/dist/index-8KnVpoZp.d.ts +32 -0
- package/dist/index-BlhJCpEk.d.ts +124 -0
- package/dist/index-CQJsFJs3.d.ts +23 -0
- package/dist/index-DK9YcR6W.d.ts +25 -0
- package/dist/index-DaNW63Mv.d.ts +21 -0
- package/dist/index-DyuT2jzP.d.ts +14 -0
- package/dist/index.d.ts +9 -165
- package/dist/index.js +44 -228
- package/dist/index.js.map +1 -1
- package/dist/ingestion/index.d.ts +6 -0
- package/dist/ingestion/index.js +7 -0
- package/dist/ingestion-CMCroK7Y.js +171 -0
- package/dist/ingestion-CMCroK7Y.js.map +1 -0
- package/dist/publisher/index.d.ts +5 -0
- package/dist/publisher/index.js +5 -0
- package/dist/publisher-DusgYvaV.js +131 -0
- package/dist/publisher-DusgYvaV.js.map +1 -0
- package/dist/server-DAeFWT4o.js +113 -0
- package/dist/server-DAeFWT4o.js.map +1 -0
- package/dist/service-B7g_lHMI.d.ts +32 -0
- package/dist/state-store/index.d.ts +4 -0
- package/dist/state-store/index.js +3 -0
- package/dist/state-store-DG_4QpWG.js +31 -0
- package/dist/state-store-DG_4QpWG.js.map +1 -0
- package/dist/telemetry-p0dE5Mdu.js +171 -0
- package/dist/telemetry-p0dE5Mdu.js.map +1 -0
- package/dist/types-ZNrA-Dn0.d.ts +160 -0
- package/dist/webhook/index.d.ts +5 -0
- package/dist/webhook/index.js +6 -0
- package/dist/webhook-BSA-tXtU.js +13 -0
- package/dist/webhook-BSA-tXtU.js.map +1 -0
- package/package.json +36 -7
- package/src/connector-app/index.ts +1 -0
- package/src/connector-app/start.ts +38 -0
- package/src/core/builder.ts +76 -9
- package/src/core/index.ts +22 -0
- package/src/core/types.ts +185 -75
- package/src/errors/index.ts +1 -0
- package/src/index.ts +8 -28
- package/src/ingestion/engine.ts +301 -179
- package/src/ingestion/index.ts +1 -0
- package/src/publisher/console.ts +18 -0
- package/src/publisher/index.ts +7 -0
- package/src/publisher/service.ts +30 -13
- package/src/publisher/wings.ts +159 -72
- package/src/runtime/context.ts +15 -0
- package/src/state-store/index.ts +2 -0
- package/src/state-store/memory.ts +25 -0
- package/src/state-store/service.ts +18 -0
- package/src/telemetry.ts +246 -0
- package/src/webhook/index.ts +8 -0
- package/src/webhook/server.ts +172 -37
- package/src/webhook/types.ts +6 -17
- package/src/ingestion/state-store.ts +0 -36
- package/src/streams/pull-stream.ts +0 -54
- package/src/streams/webhook-queue.ts +0 -14
- /package/src/{core/errors.ts → errors/connector-error.ts} +0 -0
package/README.md
CHANGED
|
@@ -1,123 +1,316 @@
|
|
|
1
1
|
# @useairfoil/connector-kit
|
|
2
2
|
|
|
3
|
-
Toolkit for building Airfoil connectors with Effect.
|
|
3
|
+
Toolkit for building Airfoil producer connectors with Effect.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
`@useairfoil/connector-kit` gives connector authors the pieces needed to define, run, test, and publish resource-first producer connectors:
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
- `Connector.define(...)` describes a connector.
|
|
8
|
+
- `Resource.entity(...)` describes a Wings table-backed resource.
|
|
9
|
+
- `Fetch.page(...)` and `Fetch.changes(...)` describe backfill and polling fetches.
|
|
10
|
+
- `Resource.webhook(...)` lets a resource normalize provider webhook payloads into mutations.
|
|
11
|
+
- `Webhook.route(...)` defines connector-level HTTP webhook routes.
|
|
12
|
+
- `Ingestion.run(...)` runs resource ingestion and checkpoints after accepted publishes.
|
|
13
|
+
- `Publisher.layerConsole` logs accepted batches locally.
|
|
14
|
+
- `Publisher.layerWings(...)` publishes resource mutations into Wings tables.
|
|
15
|
+
- `ConnectorApp.start(...)` starts a complete Node HTTP connector process.
|
|
8
16
|
|
|
9
|
-
|
|
17
|
+
The package is intentionally Layer-oriented: connector code defines resources and routes, while entrypoints provide runtime dependencies such as publishers, state stores, telemetry, and API clients.
|
|
10
18
|
|
|
11
|
-
|
|
19
|
+
## Install
|
|
12
20
|
|
|
13
21
|
```bash
|
|
14
|
-
|
|
22
|
+
pnpm add @useairfoil/connector-kit effect@beta
|
|
15
23
|
```
|
|
16
24
|
|
|
17
|
-
|
|
25
|
+
This repo currently uses Effect v4 beta. In workspace packages, prefer the workspace catalog (`"effect": "catalog:"`) so all packages stay on the pinned beta version.
|
|
26
|
+
|
|
27
|
+
## Package Shape
|
|
28
|
+
|
|
29
|
+
The package root exposes the common connector authoring API:
|
|
18
30
|
|
|
19
31
|
```ts
|
|
20
|
-
import { Schema, Effect, Layer, Stream } from "effect";
|
|
21
32
|
import {
|
|
22
|
-
|
|
23
|
-
|
|
33
|
+
Connector,
|
|
34
|
+
ConnectorApp,
|
|
35
|
+
ConnectorError,
|
|
36
|
+
Cursor,
|
|
37
|
+
Fetch,
|
|
38
|
+
Ingestion,
|
|
24
39
|
Publisher,
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
40
|
+
Resource,
|
|
41
|
+
StateStore,
|
|
42
|
+
Telemetry,
|
|
43
|
+
Webhook,
|
|
29
44
|
} from "@useairfoil/connector-kit";
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Subpath exports are available for runtime domains:
|
|
48
|
+
|
|
49
|
+
- `@useairfoil/connector-kit/ingestion`
|
|
50
|
+
- `@useairfoil/connector-kit/connector-app`
|
|
51
|
+
- `@useairfoil/connector-kit/publisher`
|
|
52
|
+
- `@useairfoil/connector-kit/state-store`
|
|
53
|
+
- `@useairfoil/connector-kit/webhook`
|
|
54
|
+
- `@useairfoil/connector-kit/errors`
|
|
55
|
+
|
|
56
|
+
Core definition helpers and `Telemetry` are intentionally root-only.
|
|
57
|
+
|
|
58
|
+
## Resource Model
|
|
59
|
+
|
|
60
|
+
Resources emit mutations, not raw rows. Connector Kit currently supports `Resource.entity(...)` for Wings table-backed resources.
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
const Posts = Resource.entity({
|
|
64
|
+
name: "posts",
|
|
65
|
+
schema: PostSchema,
|
|
66
|
+
key: "id",
|
|
67
|
+
version: "updatedAt",
|
|
68
|
+
backfill: Fetch.page({
|
|
69
|
+
pageCursor: Cursor.number(),
|
|
70
|
+
cutoff: Cursor.isoDateTime(),
|
|
71
|
+
fetch: ({ pageCursor }) =>
|
|
72
|
+
api.fetchPosts({ page: typeof pageCursor === "number" ? pageCursor : 1 }).pipe(
|
|
73
|
+
Effect.map((page) => ({
|
|
74
|
+
mutations: page.items.map(Resource.upsert),
|
|
75
|
+
nextPageCursor: page.hasMore ? page.nextPage : page.page,
|
|
76
|
+
hasMore: page.hasMore,
|
|
77
|
+
})),
|
|
78
|
+
),
|
|
79
|
+
}),
|
|
80
|
+
});
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`key` and `version` autocomplete from the decoded schema row type. Invalid field names are rejected at compile time.
|
|
84
|
+
|
|
85
|
+
Mutations are created with:
|
|
86
|
+
|
|
87
|
+
```ts
|
|
88
|
+
Resource.upsert(row);
|
|
30
89
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
created_at: Schema.Date,
|
|
90
|
+
Resource.delete({
|
|
91
|
+
key: "post_123",
|
|
92
|
+
version: "2026-01-01T00:00:00.000Z",
|
|
35
93
|
});
|
|
94
|
+
```
|
|
36
95
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
],
|
|
96
|
+
## Fetches
|
|
97
|
+
|
|
98
|
+
`Fetch.page(...)` is for paginated backfill. It receives the stored page cursor and the resource cutoff.
|
|
99
|
+
|
|
100
|
+
`Fetch.changes(...)` is for polling incremental changes. It receives the stored changes cursor and can be configured with an interval.
|
|
101
|
+
|
|
102
|
+
Cursor helpers:
|
|
103
|
+
|
|
104
|
+
- `Cursor.string()`
|
|
105
|
+
- `Cursor.number()`
|
|
106
|
+
- `Cursor.isoDateTime()`
|
|
107
|
+
- `Cursor.nowIsoDateTime`
|
|
108
|
+
|
|
109
|
+
Date cursor values are accepted at API boundaries and normalized to ISO strings before checkpoint writes.
|
|
110
|
+
|
|
111
|
+
## Webhooks
|
|
112
|
+
|
|
113
|
+
Webhooks are connector-level routes with schema-validated payloads. A route dispatches to resource-owned webhook handlers by calling `to(resource, payload)`.
|
|
114
|
+
|
|
115
|
+
```ts
|
|
116
|
+
const Posts = Resource.entity({
|
|
117
|
+
name: "posts",
|
|
118
|
+
schema: PostSchema,
|
|
119
|
+
key: "id",
|
|
120
|
+
version: "updatedAt",
|
|
121
|
+
webhook: Resource.webhook({
|
|
122
|
+
schema: PostEventSchema,
|
|
123
|
+
handler: ({ payload }) => Effect.succeed([Resource.upsert(payload.data)]),
|
|
124
|
+
}),
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
const route = Webhook.route({
|
|
128
|
+
path: "/webhooks/example",
|
|
129
|
+
ackMode: "after-publish",
|
|
130
|
+
schema: ProviderWebhookSchema,
|
|
131
|
+
handler: ({ request, rawBody, payload, to }) =>
|
|
132
|
+
Effect.gen(function* () {
|
|
133
|
+
yield* verifySignature({ request, rawBody });
|
|
134
|
+
yield* to(Posts, payload);
|
|
135
|
+
return HttpServerResponse.jsonUnsafe({ ok: true });
|
|
78
136
|
}),
|
|
79
|
-
|
|
80
|
-
|
|
137
|
+
});
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Route behavior:
|
|
141
|
+
|
|
142
|
+
- reads the raw body once
|
|
143
|
+
- parses JSON once
|
|
144
|
+
- validates the route payload schema
|
|
145
|
+
- passes `request`, `rawBody`, typed `payload`, and `to(...)` to the handler
|
|
146
|
+
- returns `400` for invalid body reads, invalid JSON, or invalid route payloads
|
|
147
|
+
- returns `500` for unexpected handler/runtime/publisher failures
|
|
81
148
|
|
|
82
|
-
|
|
149
|
+
Signature verification is connector-owned and should use `rawBody`.
|
|
150
|
+
|
|
151
|
+
`ackMode: "after-publish"` publishes collected mutations before the response is completed. `ackMode: "after-enqueue"` enqueues collected mutations and lets the background webhook consumer publish them.
|
|
152
|
+
|
|
153
|
+
## Connector Definition
|
|
154
|
+
|
|
155
|
+
```ts
|
|
156
|
+
const connector = Connector.define({
|
|
157
|
+
name: "producer-example",
|
|
158
|
+
title: "Producer Example",
|
|
159
|
+
resources: [Posts],
|
|
160
|
+
webhooks: [route],
|
|
161
|
+
});
|
|
83
162
|
```
|
|
84
163
|
|
|
85
|
-
|
|
164
|
+
## Ingestion
|
|
86
165
|
|
|
87
|
-
|
|
166
|
+
`Ingestion.run(...)` contains the runtime engine. `ConnectorApp.start(...)` is the usual entrypoint for runnable producer CLIs.
|
|
88
167
|
|
|
89
|
-
|
|
168
|
+
Current runtime behavior:
|
|
90
169
|
|
|
91
|
-
-
|
|
92
|
-
-
|
|
93
|
-
- `Publisher`
|
|
94
|
-
-
|
|
95
|
-
-
|
|
170
|
+
- runs resource backfill, changes, and webhooks concurrently
|
|
171
|
+
- initializes the same initial cutoff for backfill and changes
|
|
172
|
+
- publishes through `Publisher.Publisher`
|
|
173
|
+
- checkpoints only after an accepted publish ACK
|
|
174
|
+
- does not checkpoint rejected publishes
|
|
175
|
+
- allows empty accepted batches to advance state
|
|
176
|
+
- persists resource state through `StateStore.StateStore`
|
|
96
177
|
|
|
97
|
-
|
|
178
|
+
Use `StateStore.layerMemory` for development and tests, or provide your own durable `StateStore` implementation.
|
|
98
179
|
|
|
99
|
-
|
|
180
|
+
## Publisher
|
|
100
181
|
|
|
101
|
-
|
|
102
|
-
- a `StateStore` Layer
|
|
103
|
-
- an HTTP server Layer (if you use webhooks)
|
|
104
|
-
- any custom services your connector needs (API clients, Effect Config)
|
|
182
|
+
`Publisher.Publisher` is the Effect service boundary for publishing resource mutation batches.
|
|
105
183
|
|
|
106
|
-
|
|
184
|
+
`Publisher.layerConsole` accepts batches and logs them locally.
|
|
107
185
|
|
|
108
|
-
|
|
186
|
+
`Publisher.layerWings(...)` publishes mutations to Wings tables:
|
|
109
187
|
|
|
110
188
|
```ts
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
189
|
+
const publisherLayer = Publisher.layerWings({
|
|
190
|
+
connector,
|
|
191
|
+
tables: {
|
|
192
|
+
posts: "namespaces/default/tables/posts",
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
The `tables` map is keyed by resource name. Values can be table names or objects with a table name and optional partition value.
|
|
198
|
+
|
|
199
|
+
```ts
|
|
200
|
+
Publisher.layerWings({
|
|
201
|
+
connector,
|
|
202
|
+
tables: {
|
|
203
|
+
posts: {
|
|
204
|
+
name: "namespaces/default/tables/posts",
|
|
205
|
+
partitionValue: "account_123",
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
});
|
|
123
209
|
```
|
|
210
|
+
|
|
211
|
+
The Wings publisher resolves table metadata during layer construction, validates resource key/version/partition compatibility, sends upserts with full rows, and sends deletes with key/version-only rows.
|
|
212
|
+
|
|
213
|
+
## Minimal Example
|
|
214
|
+
|
|
215
|
+
```ts
|
|
216
|
+
import { NodeRuntime } from "@effect/platform-node";
|
|
217
|
+
import {
|
|
218
|
+
Connector,
|
|
219
|
+
ConnectorApp,
|
|
220
|
+
Cursor,
|
|
221
|
+
Fetch,
|
|
222
|
+
Publisher,
|
|
223
|
+
Resource,
|
|
224
|
+
StateStore,
|
|
225
|
+
Telemetry,
|
|
226
|
+
} from "@useairfoil/connector-kit";
|
|
227
|
+
import { ConfigProvider, Effect, Layer, Schema } from "effect";
|
|
228
|
+
import { FetchHttpClient } from "effect/unstable/http";
|
|
229
|
+
|
|
230
|
+
const PostSchema = Schema.Struct({
|
|
231
|
+
id: Schema.Number,
|
|
232
|
+
title: Schema.String,
|
|
233
|
+
updatedAt: Schema.String,
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
type Post = Schema.Schema.Type<typeof PostSchema>;
|
|
237
|
+
|
|
238
|
+
const Posts = Resource.entity({
|
|
239
|
+
name: "posts",
|
|
240
|
+
schema: PostSchema,
|
|
241
|
+
key: "id",
|
|
242
|
+
version: "updatedAt",
|
|
243
|
+
backfill: Fetch.page({
|
|
244
|
+
pageCursor: Cursor.number(),
|
|
245
|
+
cutoff: Cursor.isoDateTime(),
|
|
246
|
+
fetch: () =>
|
|
247
|
+
Effect.succeed({
|
|
248
|
+
mutations: [
|
|
249
|
+
Resource.upsert({ id: 1, title: "Hello", updatedAt: "2026-01-01T00:00:00.000Z" }),
|
|
250
|
+
],
|
|
251
|
+
nextPageCursor: 1,
|
|
252
|
+
hasMore: false,
|
|
253
|
+
}),
|
|
254
|
+
}),
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
const connector = Connector.define({
|
|
258
|
+
name: "producer-example",
|
|
259
|
+
resources: [Posts],
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
const EnvLayer = Layer.mergeAll(
|
|
263
|
+
FetchHttpClient.layer,
|
|
264
|
+
Layer.succeed(ConfigProvider.ConfigProvider, ConfigProvider.fromEnv()),
|
|
265
|
+
);
|
|
266
|
+
|
|
267
|
+
const TelemetryLayer = Telemetry.layerOtlpTracing().pipe(Layer.provide(EnvLayer));
|
|
268
|
+
const runtimeLayer = Layer.mergeAll(StateStore.layerMemory, Publisher.layerConsole, TelemetryLayer);
|
|
269
|
+
|
|
270
|
+
Effect.scoped(ConnectorApp.start(connector, { port: 8080 })).pipe(
|
|
271
|
+
Effect.provide(runtimeLayer),
|
|
272
|
+
NodeRuntime.runMain,
|
|
273
|
+
);
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
## Telemetry
|
|
277
|
+
|
|
278
|
+
`Telemetry` contains connector-kit span names, span attributes, error annotation helpers, and OTLP tracing layers.
|
|
279
|
+
|
|
280
|
+
Common entry points:
|
|
281
|
+
|
|
282
|
+
- `Telemetry.layer(config, options?)`
|
|
283
|
+
- `Telemetry.layerConfig(config, options?)`
|
|
284
|
+
- `Telemetry.layerOtlpTracing(options?)`
|
|
285
|
+
|
|
286
|
+
Default redacted headers:
|
|
287
|
+
|
|
288
|
+
- `authorization`
|
|
289
|
+
- `cookie`
|
|
290
|
+
- `set-cookie`
|
|
291
|
+
- `x-api-key`
|
|
292
|
+
- `/api[-_]?key/i`
|
|
293
|
+
- `/secret/i`
|
|
294
|
+
- `/signature/i`
|
|
295
|
+
- `/token/i`
|
|
296
|
+
|
|
297
|
+
Telemetry environment variables:
|
|
298
|
+
|
|
299
|
+
| Variable | Read by | Description |
|
|
300
|
+
| ----------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------- |
|
|
301
|
+
| `OTEL_ENABLED` | Connector Kit | Enables trace export when `true`. Defaults to `false`. |
|
|
302
|
+
| `OTEL_EXPORTER_OTLP_ENDPOINT` | Connector Kit | OTLP base URL, for example `http://localhost:4318`. Connector Kit appends `/v1/traces`. Required when `OTEL_ENABLED=true`. |
|
|
303
|
+
| `OTEL_EXPORTER_OTLP_HEADERS` | Connector Kit | Optional comma-separated headers, for example `Authorization=Bearer <token>,X-Axiom-Dataset=<dataset>`. |
|
|
304
|
+
| `OTEL_SERVICE_NAME` | Effect | Service name resource attribute, for example `producer-shopify`. |
|
|
305
|
+
| `OTEL_SERVICE_VERSION` | Effect | Optional service version resource attribute. |
|
|
306
|
+
| `OTEL_RESOURCE_ATTRIBUTES` | Effect | Optional comma-separated resource attributes, for example `deployment.environment=production,team=data`. |
|
|
307
|
+
|
|
308
|
+
## Testing
|
|
309
|
+
|
|
310
|
+
For tests, the most common setup is:
|
|
311
|
+
|
|
312
|
+
- `StateStore.layerMemory` for state
|
|
313
|
+
- a small in-memory `Publisher.Publisher` test layer
|
|
314
|
+
- `Ingestion.run(...)` or `ConnectorApp.start(...)` inside `Effect.scoped`
|
|
315
|
+
|
|
316
|
+
Keep HTTP recording concerns outside connector logic by providing a VCR-backed `HttpClient` Layer.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
//#region rolldown:runtime
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __export = (all) => {
|
|
4
|
+
let target = {};
|
|
5
|
+
for (var name in all) __defProp(target, name, {
|
|
6
|
+
get: all[name],
|
|
7
|
+
enumerable: true
|
|
8
|
+
});
|
|
9
|
+
return target;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
//#endregion
|
|
13
|
+
export { __export as t };
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import "../index-DyuT2jzP.js";
|
|
2
|
+
import "../types-ZNrA-Dn0.js";
|
|
3
|
+
import { i as start, n as App, r as StartOptions } from "../index-BlhJCpEk.js";
|
|
4
|
+
import "../service-B7g_lHMI.js";
|
|
5
|
+
import "../index-DaNW63Mv.js";
|
|
6
|
+
import "../index-8KnVpoZp.js";
|
|
7
|
+
import "../index-DK9YcR6W.js";
|
|
8
|
+
import "../index-CQJsFJs3.js";
|
|
9
|
+
export { App, StartOptions, start };
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { n as run } from "./ingestion-CMCroK7Y.js";
|
|
3
|
+
import { DateTime, Effect } from "effect";
|
|
4
|
+
import { NodeHttpServer } from "@effect/platform-node";
|
|
5
|
+
import { HttpRouter } from "effect/unstable/http";
|
|
6
|
+
import { createServer } from "node:http";
|
|
7
|
+
|
|
8
|
+
//#region src/connector-app/start.ts
|
|
9
|
+
const start = (app, options) => Effect.gen(function* () {
|
|
10
|
+
const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
|
|
11
|
+
const routes = app.webhooks?.map((route) => route.path) ?? [];
|
|
12
|
+
yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
|
|
13
|
+
port: options.port,
|
|
14
|
+
routes
|
|
15
|
+
}));
|
|
16
|
+
return yield* run(app, {
|
|
17
|
+
initialCutoff,
|
|
18
|
+
webhook: {
|
|
19
|
+
routes: app.webhooks ?? [],
|
|
20
|
+
healthPath: options.healthPath ?? "/health",
|
|
21
|
+
disableHttpLogger: options.disableHttpLogger ?? true
|
|
22
|
+
}
|
|
23
|
+
}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
//#endregion
|
|
27
|
+
//#region src/connector-app/index.ts
|
|
28
|
+
var connector_app_exports = /* @__PURE__ */ __export({ start: () => start });
|
|
29
|
+
|
|
30
|
+
//#endregion
|
|
31
|
+
export { start as n, connector_app_exports as t };
|
|
32
|
+
//# sourceMappingURL=connector-app-CvqbfN6R.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connector-app-CvqbfN6R.js","names":["runIngestion"],"sources":["../src/connector-app/start.ts","../src/connector-app/index.ts"],"sourcesContent":["import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/unstable/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n readonly port: number;\n readonly initialCutoff?: Cursor.Value;\n readonly healthPath?: HttpRouter.PathInput;\n readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n Effect.gen(function* () {\n const initialCutoff =\n options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n yield* Effect.logInfo(\"webhook server ready\").pipe(\n Effect.annotateLogs({ port: options.port, routes }),\n );\n\n return yield* runIngestion(app, {\n initialCutoff,\n webhook: {\n routes: app.webhooks ?? [],\n healthPath: options.healthPath ?? \"/health\",\n disableHttpLogger: options.disableHttpLogger ?? true,\n },\n }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n });\n","export { start, type App, type StartOptions } from \"./start\";\n"],"mappings":";;;;;;;;AAmBA,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;CACpF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,KAAK,IAAI,EAAE;AAE7D,QAAO,OAAO,QAAQ,uBAAuB,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;EAAQ,CAAC,CACpD;AAED,QAAO,OAAOA,IAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,EAAE;GAC1B,YAAY,QAAQ,cAAc;GAClC,mBAAmB,QAAQ,qBAAqB;GACjD;EACF,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,CAAC;EACnF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors-DHWylqRZ.js","names":[],"sources":["../src/errors/connector-error.ts"],"sourcesContent":["import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n readonly message: string;\n readonly cause?: unknown;\n}> {}\n"],"mappings":";;;AAEA,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { t as ConnectorError } from "./index-DyuT2jzP.js";
|
|
2
|
+
import { l as ResourceDefinition, n as ConnectorDefinition, r as Cursor, y as WebhookRoute } from "./types-ZNrA-Dn0.js";
|
|
3
|
+
import { i as Publisher } from "./service-B7g_lHMI.js";
|
|
4
|
+
import { r as StateStore } from "./index-DaNW63Mv.js";
|
|
5
|
+
import { Effect } from "effect";
|
|
6
|
+
import { HttpRouter, HttpServer } from "effect/unstable/http";
|
|
7
|
+
|
|
8
|
+
//#region src/ingestion/engine.d.ts
|
|
9
|
+
type RunBaseOptions = {
|
|
10
|
+
readonly initialCutoff?: Cursor.Value;
|
|
11
|
+
};
|
|
12
|
+
type RunOptions = RunBaseOptions & {
|
|
13
|
+
readonly webhook?: {
|
|
14
|
+
readonly routes: ReadonlyArray<WebhookRoute>;
|
|
15
|
+
readonly healthPath?: HttpRouter.PathInput;
|
|
16
|
+
readonly disableHttpLogger?: boolean;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
type RunNoWebhookOptions = RunBaseOptions & {
|
|
20
|
+
readonly webhook?: undefined;
|
|
21
|
+
};
|
|
22
|
+
type RunWebhookOptions = RunOptions & {
|
|
23
|
+
readonly webhook: NonNullable<RunOptions["webhook"]>;
|
|
24
|
+
};
|
|
25
|
+
declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options?: RunNoWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Publisher>;
|
|
26
|
+
declare function run<const Resources extends ReadonlyArray<ResourceDefinition>>(connector: ConnectorDefinition<Resources>, options: RunWebhookOptions): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;
|
|
27
|
+
declare namespace index_d_exports {
|
|
28
|
+
export { RunOptions, run };
|
|
29
|
+
}
|
|
30
|
+
//#endregion
|
|
31
|
+
export { RunOptions as n, run as r, index_d_exports as t };
|
|
32
|
+
//# sourceMappingURL=index-8KnVpoZp.d.ts.map
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { t as ConnectorError } from "./index-DyuT2jzP.js";
|
|
2
|
+
import { d as ResourceMutation, h as ResourceSchema, i as DeleteValue, l as ResourceDefinition, n as ConnectorDefinition, r as Cursor, s as PageFetch, t as ChangesFetch, v as WebhookHandler } from "./types-ZNrA-Dn0.js";
|
|
3
|
+
import { i as Publisher } from "./service-B7g_lHMI.js";
|
|
4
|
+
import { r as StateStore } from "./index-DaNW63Mv.js";
|
|
5
|
+
import { Config, Effect, Layer } from "effect";
|
|
6
|
+
import { HttpRouter } from "effect/unstable/http";
|
|
7
|
+
import * as effect_unstable_http_HttpClient0 from "effect/unstable/http/HttpClient";
|
|
8
|
+
import * as effect_unstable_http_HttpServerError0 from "effect/unstable/http/HttpServerError";
|
|
9
|
+
|
|
10
|
+
//#region src/core/builder.d.ts
|
|
11
|
+
declare const Connector: {
|
|
12
|
+
define: <const Resources extends ReadonlyArray<ResourceDefinition>>(definition: ConnectorDefinition<Resources>) => ConnectorDefinition<Resources>;
|
|
13
|
+
};
|
|
14
|
+
declare const Resource: {
|
|
15
|
+
entity: <S extends ResourceSchema, Payload = never, R = never>(definition: ResourceDefinition<S, Payload, R>) => ResourceDefinition<S, Payload, R>;
|
|
16
|
+
webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) => WebhookHandler<Row, Payload>;
|
|
17
|
+
upsert: <Row extends object>(row: Row) => ResourceMutation<Row>;
|
|
18
|
+
delete: (options: {
|
|
19
|
+
readonly key: DeleteValue;
|
|
20
|
+
readonly version: DeleteValue;
|
|
21
|
+
}) => ResourceMutation<never>;
|
|
22
|
+
};
|
|
23
|
+
declare const Fetch: {
|
|
24
|
+
page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => PageFetch<Row, R>;
|
|
25
|
+
changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => ChangesFetch<Row, R>;
|
|
26
|
+
};
|
|
27
|
+
declare const Cursor$1: {
|
|
28
|
+
string: () => Cursor.Definition<string>;
|
|
29
|
+
number: () => Cursor.Definition<number>;
|
|
30
|
+
isoDateTime: () => Cursor.Definition<string>;
|
|
31
|
+
nowIsoDateTime: Effect.Effect<string, never, never>;
|
|
32
|
+
};
|
|
33
|
+
declare namespace telemetry_d_exports {
|
|
34
|
+
export { Attr, EventAttr, EventName, OtlpTracingConfig, OtlpTracingOptions, SpanName, addCurrentSpanEvent, annotateError, layer, layerConfig, layerOtlpTracing };
|
|
35
|
+
}
|
|
36
|
+
/** Canonical span names emitted by connector-kit runtime instrumentation. */
|
|
37
|
+
declare const SpanName: {
|
|
38
|
+
readonly batchProcess: "connector.batch.process";
|
|
39
|
+
readonly webhookDecode: "connector.webhook.decode";
|
|
40
|
+
readonly webhookHandle: "connector.webhook.handle";
|
|
41
|
+
readonly publish: "connector.publish";
|
|
42
|
+
readonly stateSet: "connector.state.set";
|
|
43
|
+
readonly apiFetch: "connector.api.fetch";
|
|
44
|
+
};
|
|
45
|
+
/** Canonical span attribute names. Keep these stable for collector queries. */
|
|
46
|
+
declare const Attr: {
|
|
47
|
+
readonly connectorName: "airfoil.connector.name";
|
|
48
|
+
readonly connectorResourcesCount: "airfoil.connector.resources.count";
|
|
49
|
+
readonly resourceName: "airfoil.resource.name";
|
|
50
|
+
readonly resourceSource: "airfoil.resource.source";
|
|
51
|
+
readonly batchMutations: "airfoil.batch.mutations";
|
|
52
|
+
readonly webhookPath: "airfoil.webhook.path";
|
|
53
|
+
readonly publisherSuccess: "airfoil.publisher.success";
|
|
54
|
+
readonly stateKey: "airfoil.state.key";
|
|
55
|
+
readonly apiPath: "airfoil.api.path";
|
|
56
|
+
readonly errorPhase: "airfoil.error.phase";
|
|
57
|
+
readonly errorType: "airfoil.error.type";
|
|
58
|
+
readonly errorMessage: "airfoil.error.message";
|
|
59
|
+
readonly errorDetails: "airfoil.error.details";
|
|
60
|
+
};
|
|
61
|
+
/** Span event names for high-cardinality reproduction details. */
|
|
62
|
+
declare const EventName: {
|
|
63
|
+
readonly batchCheckpoint: "airfoil.batch.checkpoint";
|
|
64
|
+
};
|
|
65
|
+
/** Event-only attributes. These are intentionally not span attributes. */
|
|
66
|
+
declare const EventAttr: {
|
|
67
|
+
readonly batchCursor: "airfoil.batch.cursor";
|
|
68
|
+
};
|
|
69
|
+
/** Annotates the active span with phase-aware error metadata before re-failing. */
|
|
70
|
+
declare const annotateError: (phase: string, error: unknown) => Effect.Effect<void, never, never>;
|
|
71
|
+
/** Adds an event to the active span, or no-ops when no span is active. */
|
|
72
|
+
declare const addCurrentSpanEvent: (name: string, attributes?: Record<string, unknown>) => Effect.Effect<void, never, never>;
|
|
73
|
+
/** Transport configuration for OTLP trace export. All fields are direct values. */
|
|
74
|
+
type OtlpTracingConfig = {
|
|
75
|
+
readonly enabled?: boolean;
|
|
76
|
+
readonly endpoint?: string;
|
|
77
|
+
readonly headers?: Record<string, string>;
|
|
78
|
+
};
|
|
79
|
+
/** Runtime options — not config-wrappable since RegExp cannot live in Config. */
|
|
80
|
+
type OtlpTracingOptions = {
|
|
81
|
+
/** Additional connector-specific header names or regexes to redact from HTTP logs/traces. */
|
|
82
|
+
readonly redactedHeaders?: ReadonlyArray<string | RegExp>;
|
|
83
|
+
};
|
|
84
|
+
/**
|
|
85
|
+
* Trace-only OTLP export with sensitive HTTP header redaction.
|
|
86
|
+
* Takes direct config values — no `ConfigProvider` required.
|
|
87
|
+
*/
|
|
88
|
+
declare const layer: (config?: OtlpTracingConfig, options?: OtlpTracingOptions) => Layer.Layer<never, Error, effect_unstable_http_HttpClient0.HttpClient>;
|
|
89
|
+
/**
|
|
90
|
+
* Trace-only OTLP export with sensitive HTTP header redaction.
|
|
91
|
+
* Takes Effect Config-wrapped values — reads from `ConfigProvider`.
|
|
92
|
+
*
|
|
93
|
+
* @example
|
|
94
|
+
* Telemetry.layerConfig({
|
|
95
|
+
* enabled: Config.boolean("MY_OTEL_ENABLED"),
|
|
96
|
+
* endpoint: Config.string("MY_COLLECTOR_URL"),
|
|
97
|
+
* })
|
|
98
|
+
*/
|
|
99
|
+
declare const layerConfig: (config: Config.Wrap<OtlpTracingConfig>, options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
|
|
100
|
+
/**
|
|
101
|
+
* Trace-only OTLP export with sensitive HTTP header redaction.
|
|
102
|
+
* Zero-config shortcut that reads the standard `OTEL_*` env var names.
|
|
103
|
+
*
|
|
104
|
+
* Reads `OTEL_ENABLED`, `OTEL_EXPORTER_OTLP_ENDPOINT`, and
|
|
105
|
+
* `OTEL_EXPORTER_OTLP_HEADERS`. Effect's OTLP resource handling reads
|
|
106
|
+
* `OTEL_SERVICE_NAME`, `OTEL_SERVICE_VERSION`, and `OTEL_RESOURCE_ATTRIBUTES`.
|
|
107
|
+
*/
|
|
108
|
+
declare const layerOtlpTracing: (options?: OtlpTracingOptions) => Layer.Layer<never, Error | Config.ConfigError, effect_unstable_http_HttpClient0.HttpClient>;
|
|
109
|
+
//#endregion
|
|
110
|
+
//#region src/connector-app/start.d.ts
|
|
111
|
+
type App = ConnectorDefinition;
|
|
112
|
+
type StartOptions = {
|
|
113
|
+
readonly port: number;
|
|
114
|
+
readonly initialCutoff?: Cursor.Value;
|
|
115
|
+
readonly healthPath?: HttpRouter.PathInput;
|
|
116
|
+
readonly disableHttpLogger?: boolean;
|
|
117
|
+
};
|
|
118
|
+
declare const start: (app: App, options: StartOptions) => Effect.Effect<void, ConnectorError | effect_unstable_http_HttpServerError0.ServeError, StateStore | Publisher>;
|
|
119
|
+
declare namespace index_d_exports {
|
|
120
|
+
export { App, StartOptions, start };
|
|
121
|
+
}
|
|
122
|
+
//#endregion
|
|
123
|
+
export { telemetry_d_exports as a, Fetch as c, start as i, Resource as l, App as n, Connector as o, StartOptions as r, Cursor$1 as s, index_d_exports as t };
|
|
124
|
+
//# sourceMappingURL=index-BlhJCpEk.d.ts.map
|