@useairfoil/connector-kit 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.txt +201 -0
- package/METRICS.md +38 -42
- package/README.md +111 -444
- package/dist/annotations-vzSzXtr-.d.mts +34 -0
- package/dist/arrow-flight-DDQ0ktx9.mjs +251 -0
- package/dist/arrow-flight-DDQ0ktx9.mjs.map +1 -0
- package/dist/auth/index.d.mts +2 -0
- package/dist/{auth-DvU_CZ7C.js → auth/index.mjs} +6 -8
- package/dist/auth/index.mjs.map +1 -0
- package/dist/connector-app/index.d.mts +2 -0
- package/dist/connector-app/index.mjs +2 -0
- package/dist/connector-app-C6_dZnhE.mjs +256 -0
- package/dist/connector-app-C6_dZnhE.mjs.map +1 -0
- package/dist/{constants-D113Ut1x.js → constants-v3UX-hf6.mjs} +5 -5
- package/dist/constants-v3UX-hf6.mjs.map +1 -0
- package/dist/demo/index.d.mts +2 -0
- package/dist/demo/index.mjs +2 -0
- package/dist/demo-DqTJ7TWI.mjs +204 -0
- package/dist/demo-DqTJ7TWI.mjs.map +1 -0
- package/dist/errors/index.d.mts +2 -0
- package/dist/errors/index.mjs +2 -0
- package/dist/{errors-DHWylqRZ.js → errors-pT0YVJU5.mjs} +2 -3
- package/dist/errors-pT0YVJU5.mjs.map +1 -0
- package/dist/iceberg/index.d.mts +3 -0
- package/dist/iceberg/index.mjs +3 -0
- package/dist/iceberg-Cj2PwlNm.mjs +30 -0
- package/dist/iceberg-Cj2PwlNm.mjs.map +1 -0
- package/dist/index-BTAbexBp.d.mts +55 -0
- package/dist/{index-DXa26Ry5.d.ts → index-BX8G7d41.d.mts} +1 -2
- package/dist/{index-VpHaZXZw.d.ts → index-Byr8NTdG.d.mts} +11 -13
- package/dist/index-CV6V6fyx.d.mts +79 -0
- package/dist/{index-C5Ma7V7t.d.ts → index-CZKx2Yce.d.mts} +72 -37
- package/dist/{index-DYpEMJf-.d.ts → index-CgvFomO6.d.mts} +6 -7
- package/dist/{index-bfjH7Yb0.d.ts → index-DeTwIUKl.d.mts} +9 -10
- package/dist/{index-DKoRwyFn.d.ts → index-DkVNO6ks.d.mts} +2 -5
- package/dist/{index-BKw7jA2a.d.ts → index-dHS59EZl.d.mts} +10 -8
- package/dist/{index-CR9HJDdr.d.ts → index-eFuoMZZ3.d.mts} +18 -20
- package/dist/{index-ZSuiZX5H.d.ts → index-fHIJJoRe.d.mts} +25 -30
- package/dist/index-uFa2wwYo.d.mts +27 -0
- package/dist/index.d.mts +16 -0
- package/dist/index.mjs +45 -0
- package/dist/index.mjs.map +1 -0
- package/dist/ingestion/index.d.mts +2 -0
- package/dist/ingestion/index.mjs +2 -0
- package/dist/ingestion-DoCUwm5N.mjs +275 -0
- package/dist/ingestion-DoCUwm5N.mjs.map +1 -0
- package/dist/ingestor/index.d.mts +3 -0
- package/dist/ingestor/index.mjs +4 -0
- package/dist/ingestor-DjCEzBzZ.mjs +26 -0
- package/dist/ingestor-DjCEzBzZ.mjs.map +1 -0
- package/dist/instrumented-Bq1WX2L5.mjs +29 -0
- package/dist/instrumented-Bq1WX2L5.mjs.map +1 -0
- package/dist/manifest/index.d.mts +2 -0
- package/dist/manifest/{index.js → index.mjs} +2 -4
- package/dist/{manifest-P7kBnch9.js → manifest-CABeIhAG.mjs} +14 -19
- package/dist/manifest-CABeIhAG.mjs.map +1 -0
- package/dist/{telemetry-j_nxwYfh.js → metrics-C901IIYZ.mjs} +172 -26
- package/dist/metrics-C901IIYZ.mjs.map +1 -0
- package/dist/{metrics-C4V1pm6v.d.ts → metrics-DoJ0WeKu.d.mts} +18 -13
- package/dist/metrics.d.mts +2 -0
- package/dist/metrics.mjs +2 -0
- package/dist/{chunk-Bp6m_JJh.js → rolldown-runtime-D7D4PA-g.mjs} +4 -4
- package/dist/runtime-config/index.d.mts +2 -0
- package/dist/runtime-config/index.mjs +3 -0
- package/dist/runtime-config-Bn8Fivt1.mjs +100 -0
- package/dist/runtime-config-Bn8Fivt1.mjs.map +1 -0
- package/dist/server-D1TTN0_D.mjs +144 -0
- package/dist/server-D1TTN0_D.mjs.map +1 -0
- package/dist/service-BRoy_EId.d.mts +27 -0
- package/dist/service-jWxxfEht.mjs +7 -0
- package/dist/service-jWxxfEht.mjs.map +1 -0
- package/dist/state-store/index.d.mts +2 -0
- package/dist/state-store/index.mjs +2 -0
- package/dist/{state-store-BdYulQOY.js → state-store-DhSrKVEH.mjs} +15 -22
- package/dist/state-store-DhSrKVEH.mjs.map +1 -0
- package/dist/status/{index.d.ts → index.d.mts} +1 -4
- package/dist/status/{index.js → index.mjs} +2 -6
- package/dist/{status-BTTd7AUR.js → status-4h6Oxh-O.mjs} +14 -16
- package/dist/status-4h6Oxh-O.mjs.map +1 -0
- package/dist/table-CxM09xf7.mjs +307 -0
- package/dist/table-CxM09xf7.mjs.map +1 -0
- package/dist/types-NtSv6gwl.d.mts +220 -0
- package/dist/webhook/index.d.mts +3 -0
- package/dist/webhook/index.mjs +12 -0
- package/dist/webhook/index.mjs.map +1 -0
- package/package.json +72 -58
- package/src/catalog/retry.ts +25 -0
- package/src/catalog/table.ts +237 -0
- package/src/connector-app/create-table.ts +47 -0
- package/src/connector-app/demo.ts +47 -0
- package/src/connector-app/index.ts +3 -0
- package/src/connector-app/setup.ts +69 -0
- package/src/connector-app/start.ts +1 -1
- package/src/core/builder.ts +15 -23
- package/src/core/index.ts +11 -1
- package/src/core/types.ts +114 -39
- package/src/demo/definition.ts +68 -0
- package/src/demo/generate.ts +167 -0
- package/src/demo/index.ts +3 -0
- package/src/demo/run.ts +172 -0
- package/src/iceberg/annotations.ts +49 -0
- package/src/iceberg/error.ts +9 -0
- package/src/iceberg/index.ts +3 -0
- package/src/iceberg/table.ts +461 -0
- package/src/index.ts +3 -1
- package/src/ingestion/engine.ts +240 -60
- package/src/ingestor/arrow-flight.ts +141 -0
- package/src/ingestor/arrow.ts +248 -0
- package/src/ingestor/console.ts +13 -0
- package/src/ingestor/index.ts +13 -0
- package/src/ingestor/instrumented.ts +61 -0
- package/src/ingestor/service.ts +30 -0
- package/src/manifest/config.ts +3 -3
- package/src/metrics.ts +66 -20
- package/src/runtime-config/config.ts +11 -8
- package/src/runtime-config/constants.ts +3 -2
- package/src/runtime-config/index.ts +2 -7
- package/src/runtime-config/layer.ts +1 -1
- package/src/runtime-config/table-bindings.ts +50 -136
- package/src/state-store/layer-key-value-store.ts +8 -3
- package/src/state-store/layer-memory.ts +1 -1
- package/src/state-store/layer-sql.ts +4 -4
- package/src/state-store/schema.ts +2 -2
- package/src/state-store/state.ts +14 -2
- package/src/status/route.ts +2 -2
- package/src/telemetry.ts +20 -28
- package/src/webhook/server.ts +39 -40
- package/dist/auth/index.d.ts +0 -2
- package/dist/auth/index.js +0 -3
- package/dist/auth-DvU_CZ7C.js.map +0 -1
- package/dist/connector-app/index.d.ts +0 -14
- package/dist/connector-app/index.js +0 -12
- package/dist/connector-app-BljHd0Pe.js +0 -64
- package/dist/connector-app-BljHd0Pe.js.map +0 -1
- package/dist/constants-D113Ut1x.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.js +0 -3
- package/dist/errors-DHWylqRZ.js.map +0 -1
- package/dist/index-BA8ASHKr.d.ts +0 -31
- package/dist/index.d.ts +0 -14
- package/dist/index.js +0 -60
- package/dist/index.js.map +0 -1
- package/dist/ingestion/index.d.ts +0 -6
- package/dist/ingestion/index.js +0 -11
- package/dist/ingestion-QBGKrFr9.js +0 -200
- package/dist/ingestion-QBGKrFr9.js.map +0 -1
- package/dist/manifest/index.d.ts +0 -2
- package/dist/manifest-P7kBnch9.js.map +0 -1
- package/dist/metrics-inkCAxKq.js +0 -136
- package/dist/metrics-inkCAxKq.js.map +0 -1
- package/dist/metrics.d.ts +0 -4
- package/dist/metrics.js +0 -5
- package/dist/publisher/index.d.ts +0 -6
- package/dist/publisher/index.js +0 -8
- package/dist/publisher-BdWJOfer.js +0 -138
- package/dist/publisher-BdWJOfer.js.map +0 -1
- package/dist/runtime-config/index.d.ts +0 -4
- package/dist/runtime-config/index.js +0 -5
- package/dist/runtime-config-IbyCVpei.js +0 -60
- package/dist/runtime-config-IbyCVpei.js.map +0 -1
- package/dist/server-D-eLX3yN.js +0 -183
- package/dist/server-D-eLX3yN.js.map +0 -1
- package/dist/service-CUjGhe-7.d.ts +0 -32
- package/dist/service-DiP-XETo.js +0 -8
- package/dist/service-DiP-XETo.js.map +0 -1
- package/dist/state-store/index.d.ts +0 -3
- package/dist/state-store/index.js +0 -5
- package/dist/state-store-BdYulQOY.js.map +0 -1
- package/dist/status-BTTd7AUR.js.map +0 -1
- package/dist/table-bindings-Dp1DhTeS.js +0 -106
- package/dist/table-bindings-Dp1DhTeS.js.map +0 -1
- package/dist/telemetry-j_nxwYfh.js.map +0 -1
- package/dist/types-jVD5Ys43.d.ts +0 -174
- package/dist/webhook/index.d.ts +0 -7
- package/dist/webhook/index.js +0 -9
- package/dist/webhook-MhFcLW-o.js +0 -13
- package/dist/webhook-MhFcLW-o.js.map +0 -1
- package/src/publisher/console.ts +0 -18
- package/src/publisher/index.ts +0 -7
- package/src/publisher/instrumented.ts +0 -75
- package/src/publisher/service.ts +0 -37
- package/src/publisher/wings.ts +0 -204
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import "../index-DXa26Ry5.js";
|
|
2
|
-
import "../index-DKoRwyFn.js";
|
|
3
|
-
import "../types-jVD5Ys43.js";
|
|
4
|
-
import { d as CheckResult, f as ResourceCheckResult, i as start, n as App, p as check, r as StartOptions, u as CheckOptions } from "../index-C5Ma7V7t.js";
|
|
5
|
-
import "../service-CUjGhe-7.js";
|
|
6
|
-
import "../index-DYpEMJf-.js";
|
|
7
|
-
import "../index-bfjH7Yb0.js";
|
|
8
|
-
import "../index-CR9HJDdr.js";
|
|
9
|
-
import "../metrics-C4V1pm6v.js";
|
|
10
|
-
import "../index-ZSuiZX5H.js";
|
|
11
|
-
import "../index-BA8ASHKr.js";
|
|
12
|
-
import "../index-VpHaZXZw.js";
|
|
13
|
-
import "../index-BKw7jA2a.js";
|
|
14
|
-
export { App, CheckOptions, CheckResult, ResourceCheckResult, StartOptions, check, start };
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import "../errors-DHWylqRZ.js";
|
|
2
|
-
import { n as start, r as check } from "../connector-app-BljHd0Pe.js";
|
|
3
|
-
import "../constants-D113Ut1x.js";
|
|
4
|
-
import "../telemetry-j_nxwYfh.js";
|
|
5
|
-
import "../metrics-inkCAxKq.js";
|
|
6
|
-
import "../service-DiP-XETo.js";
|
|
7
|
-
import "../server-D-eLX3yN.js";
|
|
8
|
-
import "../state-store-BdYulQOY.js";
|
|
9
|
-
import "../status-BTTd7AUR.js";
|
|
10
|
-
import "../ingestion-QBGKrFr9.js";
|
|
11
|
-
|
|
12
|
-
export { check, start };
|
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
-
import { n as run } from "./ingestion-QBGKrFr9.js";
|
|
3
|
-
import { Context, DateTime, Effect, Layer, Tuple } from "effect";
|
|
4
|
-
import { NodeHttpServer } from "@effect/platform-node";
|
|
5
|
-
import { HttpRouter } from "effect/unstable/http";
|
|
6
|
-
import { createServer } from "node:http";
|
|
7
|
-
|
|
8
|
-
//#region src/connector-app/check.ts
|
|
9
|
-
const ok = { _tag: "ok" };
|
|
10
|
-
const error = (message) => ({
|
|
11
|
-
_tag: "error",
|
|
12
|
-
message
|
|
13
|
-
});
|
|
14
|
-
const fromEntries = (entries) => Object.fromEntries(entries);
|
|
15
|
-
const resultsFor = (names, result) => fromEntries(names.map((name) => Tuple.make(name, result)));
|
|
16
|
-
/**
|
|
17
|
-
* Builds the connector service from its layer and validates only the selected
|
|
18
|
-
* resources. The layer remains open until every check finishes.
|
|
19
|
-
*/
|
|
20
|
-
function check(service, layer, options) {
|
|
21
|
-
if (options.resources.length === 0) return Effect.succeed(fromEntries([]));
|
|
22
|
-
return service.pipe(Effect.flatMap((connector) => Effect.forEach(options.resources, (name) => {
|
|
23
|
-
const resource = connector.resources.find((candidate) => candidate.name === name);
|
|
24
|
-
return (resource ? resource.check.pipe(Effect.match({
|
|
25
|
-
onFailure: (failure) => error(failure.message),
|
|
26
|
-
onSuccess: () => ok
|
|
27
|
-
})) : Effect.succeed(error(`Unknown connector resource: ${name}`))).pipe(Effect.map((result) => Tuple.make(name, result)));
|
|
28
|
-
}).pipe(Effect.map(fromEntries))), Effect.provide(layer), Effect.match({
|
|
29
|
-
onFailure: (failure) => resultsFor(options.resources, error(failure.message)),
|
|
30
|
-
onSuccess: (results) => results
|
|
31
|
-
}));
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
//#endregion
|
|
35
|
-
//#region src/connector-app/start.ts
|
|
36
|
-
const start = (app, options) => Effect.gen(function* () {
|
|
37
|
-
const initialCutoff = options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
|
|
38
|
-
const routes = app.webhooks?.map((route) => route.path) ?? [];
|
|
39
|
-
yield* Effect.logInfo("webhook server ready").pipe(Effect.annotateLogs({
|
|
40
|
-
port: options.port,
|
|
41
|
-
routes
|
|
42
|
-
}));
|
|
43
|
-
return yield* run(app, {
|
|
44
|
-
initialCutoff,
|
|
45
|
-
webhook: {
|
|
46
|
-
routes: app.webhooks ?? [],
|
|
47
|
-
healthPath: options.healthPath ?? "/health",
|
|
48
|
-
metricsPath: options.metricsPath,
|
|
49
|
-
statusPath: options.statusPath,
|
|
50
|
-
disableHttpLogger: options.disableHttpLogger ?? true
|
|
51
|
-
}
|
|
52
|
-
}).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));
|
|
53
|
-
});
|
|
54
|
-
|
|
55
|
-
//#endregion
|
|
56
|
-
//#region src/connector-app/index.ts
|
|
57
|
-
var connector_app_exports = /* @__PURE__ */ __export({
|
|
58
|
-
check: () => check,
|
|
59
|
-
start: () => start
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
//#endregion
|
|
63
|
-
export { start as n, check as r, connector_app_exports as t };
|
|
64
|
-
//# sourceMappingURL=connector-app-BljHd0Pe.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"connector-app-BljHd0Pe.js","names":["ok: ResourceCheckResult","runIngestion"],"sources":["../src/connector-app/check.ts","../src/connector-app/start.ts","../src/connector-app/index.ts"],"sourcesContent":["import { Context, Effect, Layer, Tuple } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition, ResourceName } from \"../core/types\";\n\nexport type ResourceCheckResult =\n | { readonly _tag: \"ok\" }\n | { readonly _tag: \"error\"; readonly message: string };\n\nexport type CheckOptions<Name extends string> = {\n readonly resources: ReadonlyArray<Name>;\n};\n\ntype ResourceCheckResults<Name extends PropertyKey> = {\n readonly [Key in Name]: ResourceCheckResult;\n};\n\nexport type CheckResult<\n Resources extends ReadonlyArray<ResourceDefinition>,\n Name extends ResourceName<Resources> = ResourceName<Resources>,\n> = ResourceCheckResults<Name>;\n\ntype CheckFailure = { readonly message: string };\n\nconst ok: ResourceCheckResult = { _tag: \"ok\" };\n\nconst error = (message: string): ResourceCheckResult => ({ _tag: \"error\", message });\n\nconst fromEntries = <Name extends string>(\n entries: ReadonlyArray<readonly [Name, ResourceCheckResult]>,\n): ResourceCheckResults<Name> => Object.fromEntries(entries) as ResourceCheckResults<Name>;\n\nconst resultsFor = <Name extends string>(names: ReadonlyArray<Name>, result: ResourceCheckResult) =>\n fromEntries(names.map((name) => Tuple.make(name, result)));\n\n/**\n * Builds the connector service from its layer and validates only the selected\n * resources. The layer remains open until every check finishes.\n */\nexport function check<\n Identifier,\n const Resources extends ReadonlyArray<ResourceDefinition>,\n E extends CheckFailure,\n R,\n const Name extends ResourceName<Resources>,\n>(\n service: Context.Key<Identifier, ConnectorDefinition<Resources>>,\n layer: Layer.Layer<Identifier, E, R>,\n options: CheckOptions<Name>,\n): Effect.Effect<CheckResult<Resources, Name>, never, R> {\n if (options.resources.length === 0) {\n return Effect.succeed(fromEntries<Name>([]));\n }\n\n return service.pipe(\n Effect.flatMap((connector) =>\n Effect.forEach(options.resources, (name) => {\n const resource = connector.resources.find((candidate) => candidate.name === name);\n const result = resource\n ? resource.check.pipe(\n Effect.match({\n onFailure: (failure) => error(failure.message),\n onSuccess: () => ok,\n }),\n )\n : Effect.succeed(error(`Unknown connector resource: ${name}`));\n\n return result.pipe(Effect.map((result) => Tuple.make(name, result)));\n }).pipe(Effect.map(fromEntries)),\n ),\n Effect.provide(layer),\n Effect.match({\n onFailure: (failure) => resultsFor(options.resources, error(failure.message)),\n onSuccess: (results) => results,\n }),\n );\n}\n","import { NodeHttpServer } from \"@effect/platform-node\";\nimport { DateTime, Effect } from \"effect\";\nimport { HttpRouter } from \"effect/unstable/http\";\nimport { createServer } from \"node:http\";\n\nimport type { ConnectorDefinition } from \"../core\";\nimport type { Cursor } from \"../core/types\";\n\nimport { run as runIngestion } from \"../ingestion\";\n\nexport type App = ConnectorDefinition;\n\nexport type StartOptions = {\n readonly port: number;\n readonly initialCutoff?: Cursor.Value;\n readonly healthPath?: HttpRouter.PathInput;\n readonly metricsPath?: HttpRouter.PathInput | false;\n readonly statusPath?: HttpRouter.PathInput | false;\n readonly disableHttpLogger?: boolean;\n};\n\nexport const start = (app: App, options: StartOptions) =>\n Effect.gen(function* () {\n const initialCutoff =\n options.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n const routes = app.webhooks?.map((route) => route.path) ?? [];\n\n yield* Effect.logInfo(\"webhook server ready\").pipe(\n Effect.annotateLogs({ port: options.port, routes }),\n );\n\n return yield* runIngestion(app, {\n initialCutoff,\n webhook: {\n routes: app.webhooks ?? [],\n healthPath: options.healthPath ?? \"/health\",\n metricsPath: options.metricsPath,\n statusPath: options.statusPath,\n disableHttpLogger: options.disableHttpLogger ?? true,\n },\n }).pipe(Effect.provide(NodeHttpServer.layer(createServer, { port: options.port })));\n });\n","export { check, type CheckOptions, type CheckResult, type ResourceCheckResult } from \"./check\";\nexport { start, type App, type StartOptions } from \"./start\";\n"],"mappings":";;;;;;;;AAuBA,MAAMA,KAA0B,EAAE,MAAM,MAAM;AAE9C,MAAM,SAAS,aAA0C;CAAE,MAAM;CAAS;CAAS;AAEnF,MAAM,eACJ,YAC+B,OAAO,YAAY,QAAQ;AAE5D,MAAM,cAAmC,OAA4B,WACnE,YAAY,MAAM,KAAK,SAAS,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC;;;;;AAM5D,SAAgB,MAOd,SACA,OACA,SACuD;AACvD,KAAI,QAAQ,UAAU,WAAW,EAC/B,QAAO,OAAO,QAAQ,YAAkB,EAAE,CAAC,CAAC;AAG9C,QAAO,QAAQ,KACb,OAAO,SAAS,cACd,OAAO,QAAQ,QAAQ,YAAY,SAAS;EAC1C,MAAM,WAAW,UAAU,UAAU,MAAM,cAAc,UAAU,SAAS,KAAK;AAUjF,UATe,WACX,SAAS,MAAM,KACb,OAAO,MAAM;GACX,YAAY,YAAY,MAAM,QAAQ,QAAQ;GAC9C,iBAAiB;GAClB,CAAC,CACH,GACD,OAAO,QAAQ,MAAM,+BAA+B,OAAO,CAAC,EAElD,KAAK,OAAO,KAAK,WAAW,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC;GACpE,CAAC,KAAK,OAAO,IAAI,YAAY,CAAC,CACjC,EACD,OAAO,QAAQ,MAAM,EACrB,OAAO,MAAM;EACX,YAAY,YAAY,WAAW,QAAQ,WAAW,MAAM,QAAQ,QAAQ,CAAC;EAC7E,YAAY,YAAY;EACzB,CAAC,CACH;;;;;ACrDH,MAAa,SAAS,KAAU,YAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,gBACJ,QAAQ,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;CACpF,MAAM,SAAS,IAAI,UAAU,KAAK,UAAU,MAAM,KAAK,IAAI,EAAE;AAE7D,QAAO,OAAO,QAAQ,uBAAuB,CAAC,KAC5C,OAAO,aAAa;EAAE,MAAM,QAAQ;EAAM;EAAQ,CAAC,CACpD;AAED,QAAO,OAAOC,IAAa,KAAK;EAC9B;EACA,SAAS;GACP,QAAQ,IAAI,YAAY,EAAE;GAC1B,YAAY,QAAQ,cAAc;GAClC,aAAa,QAAQ;GACrB,YAAY,QAAQ;GACpB,mBAAmB,QAAQ,qBAAqB;GACjD;EACF,CAAC,CAAC,KAAK,OAAO,QAAQ,eAAe,MAAM,cAAc,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,CAAC;EACnF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"constants-D113Ut1x.js","names":[],"sources":["../src/runtime-config/constants.ts"],"sourcesContent":["import { HashSet } from \"effect\";\n\n/** Shared runtime keys owned by Airfoil rather than an individual connector. */\nexport const PlatformRuntimeKey = {\n configPath: \"AIRFOIL_CONFIG_PATH\",\n httpPort: \"AIRFOIL_HTTP_PORT\",\n tableBindings: \"AIRFOIL_TABLE_BINDINGS\",\n connectorInstanceId: \"AIRFOIL_CONNECTOR_INSTANCE_ID\",\n stateTable: \"AIRFOIL_STATE_TABLE\",\n postgresConnectionString: \"POSTGRES_CONNECTION_STRING\",\n wingsHost: \"WINGS_HOST\",\n wingsNamespace: \"WINGS_NAMESPACE\",\n otelEnabled: \"OTEL_ENABLED\",\n otelExporterOtlpEndpoint: \"OTEL_EXPORTER_OTLP_ENDPOINT\",\n otelExporterOtlpHeaders: \"OTEL_EXPORTER_OTLP_HEADERS\",\n otelServiceName: \"OTEL_SERVICE_NAME\",\n otelServiceVersion: \"OTEL_SERVICE_VERSION\",\n otelResourceAttributes: \"OTEL_RESOURCE_ATTRIBUTES\",\n} as const;\n\n/** Defaults for optional platform-owned runtime configuration. */\nexport const PlatformRuntimeDefault = {\n httpPort: 8080,\n stateTable: \"_airfoil_connectors_state\",\n} as const;\n\nconst platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));\n\n/** @internal */\nexport const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);\n"],"mappings":";;;;AAGA,MAAa,qBAAqB;CAChC,YAAY;CACZ,UAAU;CACV,eAAe;CACf,qBAAqB;CACrB,YAAY;CACZ,0BAA0B;CAC1B,WAAW;CACX,gBAAgB;CAChB,aAAa;CACb,0BAA0B;CAC1B,yBAAyB;CACzB,iBAAiB;CACjB,oBAAoB;CACpB,wBAAwB;CACzB;;AAGD,MAAa,yBAAyB;CACpC,UAAU;CACV,YAAY;CACb;AAED,MAAM,sBAAsB,QAAQ,aAAa,OAAO,OAAO,mBAAmB,CAAC;;AAGnF,MAAa,wBAAwB,QAAyB,QAAQ,IAAI,qBAAqB,IAAI"}
|
package/dist/errors/index.d.ts
DELETED
package/dist/errors/index.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"errors-DHWylqRZ.js","names":[],"sources":["../src/errors/connector-error.ts"],"sourcesContent":["import { Data } from \"effect\";\n\nexport class ConnectorError extends Data.TaggedError(\"ConnectorError\")<{\n readonly message: string;\n readonly cause?: unknown;\n}> {}\n"],"mappings":";;;AAEA,IAAa,iBAAb,cAAoC,KAAK,YAAY,iBAAiB,CAGnE"}
|
package/dist/index-BA8ASHKr.d.ts
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import { t as ConnectorError } from "./index-DKoRwyFn.js";
|
|
2
|
-
import { c as ResourceBatch, n as ConnectorDefinition } from "./types-jVD5Ys43.js";
|
|
3
|
-
import { a as PublisherService, i as Publisher, n as PublishOptions, r as PublishSource, t as PublishAck } from "./service-CUjGhe-7.js";
|
|
4
|
-
import { d as RuntimeConfigError } from "./index-ZSuiZX5H.js";
|
|
5
|
-
import { Layer } from "effect";
|
|
6
|
-
import * as Wings from "@useairfoil/wings";
|
|
7
|
-
|
|
8
|
-
//#region src/publisher/wings.d.ts
|
|
9
|
-
/** Programmatic mapping between a connector resource and a Wings table. */
|
|
10
|
-
type WingsTableMapping = {
|
|
11
|
-
readonly name: string;
|
|
12
|
-
readonly partitionValue?: Wings.PartitionValue.PartitionValue;
|
|
13
|
-
};
|
|
14
|
-
/** Direct Wings publisher configuration, mainly for tests and local runtimes. */
|
|
15
|
-
type WingsPublisherConfig = {
|
|
16
|
-
readonly connector: ConnectorDefinition;
|
|
17
|
-
readonly tables: Record<string, string | WingsTableMapping>;
|
|
18
|
-
};
|
|
19
|
-
/** Builds a Wings publisher from explicit table mappings. */
|
|
20
|
-
declare const layerWings: (config: WingsPublisherConfig) => Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient>;
|
|
21
|
-
/** Builds a Wings publisher from the complete hosted `AIRFOIL_TABLE_BINDINGS` document. */
|
|
22
|
-
declare const layerWingsConfig: (connector: ConnectorDefinition) => Layer.Layer<Publisher, ConnectorError | RuntimeConfigError, Wings.WingsClient.WingsClient>;
|
|
23
|
-
//#endregion
|
|
24
|
-
//#region src/publisher/console.d.ts
|
|
25
|
-
declare const layerConsole: Layer.Layer<Publisher, never, never>;
|
|
26
|
-
declare namespace index_d_exports {
|
|
27
|
-
export { PublishAck, PublishOptions, PublishSource, Publisher, PublisherService, ResourceBatch, WingsPublisherConfig, layerConsole, layerWings, layerWingsConfig };
|
|
28
|
-
}
|
|
29
|
-
//#endregion
|
|
30
|
-
export { layerWingsConfig as a, layerWings as i, layerConsole as n, WingsPublisherConfig as r, index_d_exports as t };
|
|
31
|
-
//# sourceMappingURL=index-BA8ASHKr.d.ts.map
|
package/dist/index.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { t as index_d_exports } from "./index-DXa26Ry5.js";
|
|
2
|
-
import { t as ConnectorError } from "./index-DKoRwyFn.js";
|
|
3
|
-
import { S as WebhookRouteContext, _ as ResourceState, a as FetchChangesResult, b as WebhookHandler, c as ResourceBatch, d as ResourceMutation, g as ResourceSchema, h as ResourceRows, l as ResourceDefinition, m as ResourceRow, n as ConnectorDefinition, o as FetchPageResult, p as ResourcePayload, r as Cursor$1, s as PageFetch, t as ChangesFetch, u as ResourceField, x as WebhookRoute, y as WebhookAckMode } from "./types-jVD5Ys43.js";
|
|
4
|
-
import { a as telemetry_d_exports, c as Fetch, l as Resource, o as Connector, s as Cursor, t as index_d_exports$1 } from "./index-C5Ma7V7t.js";
|
|
5
|
-
import "./service-CUjGhe-7.js";
|
|
6
|
-
import { t as index_d_exports$6 } from "./index-DYpEMJf-.js";
|
|
7
|
-
import { t as index_d_exports$2 } from "./index-bfjH7Yb0.js";
|
|
8
|
-
import { t as index_d_exports$3 } from "./index-CR9HJDdr.js";
|
|
9
|
-
import { g as metrics_d_exports } from "./metrics-C4V1pm6v.js";
|
|
10
|
-
import { t as index_d_exports$5 } from "./index-ZSuiZX5H.js";
|
|
11
|
-
import { t as index_d_exports$4 } from "./index-BA8ASHKr.js";
|
|
12
|
-
import { t as index_d_exports$7 } from "./index-VpHaZXZw.js";
|
|
13
|
-
import { t as index_d_exports$8 } from "./index-BKw7jA2a.js";
|
|
14
|
-
export { index_d_exports as Auth, ChangesFetch, Connector, index_d_exports$1 as ConnectorApp, ConnectorDefinition, ConnectorError, Cursor, Cursor$1 as CursorTypes, Fetch, FetchChangesResult, FetchPageResult, index_d_exports$2 as Ingestion, index_d_exports$3 as Manifest, metrics_d_exports as Metrics, PageFetch, index_d_exports$4 as Publisher, Resource, ResourceBatch, ResourceDefinition, ResourceField, ResourceMutation, ResourcePayload, ResourceRow, ResourceRows, ResourceSchema, ResourceState, index_d_exports$5 as RuntimeConfig, index_d_exports$6 as StateStore, index_d_exports$7 as Status, telemetry_d_exports as Telemetry, index_d_exports$8 as Webhook, WebhookAckMode, WebhookHandler, WebhookRoute, WebhookRouteContext };
|
package/dist/index.js
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
import { t as ConnectorError } from "./errors-DHWylqRZ.js";
|
|
2
|
-
import { t as auth_exports } from "./auth-DvU_CZ7C.js";
|
|
3
|
-
import { t as connector_app_exports } from "./connector-app-BljHd0Pe.js";
|
|
4
|
-
import "./constants-D113Ut1x.js";
|
|
5
|
-
import { s as telemetry_exports } from "./telemetry-j_nxwYfh.js";
|
|
6
|
-
import { s as metrics_exports } from "./metrics-inkCAxKq.js";
|
|
7
|
-
import "./service-DiP-XETo.js";
|
|
8
|
-
import "./server-D-eLX3yN.js";
|
|
9
|
-
import { t as state_store_exports } from "./state-store-BdYulQOY.js";
|
|
10
|
-
import { t as status_exports } from "./status-BTTd7AUR.js";
|
|
11
|
-
import { t as ingestion_exports } from "./ingestion-QBGKrFr9.js";
|
|
12
|
-
import { t as manifest_exports } from "./manifest-P7kBnch9.js";
|
|
13
|
-
import { t as publisher_exports } from "./publisher-BdWJOfer.js";
|
|
14
|
-
import "./table-bindings-Dp1DhTeS.js";
|
|
15
|
-
import { t as runtime_config_exports } from "./runtime-config-IbyCVpei.js";
|
|
16
|
-
import { n as webhook_exports } from "./webhook-MhFcLW-o.js";
|
|
17
|
-
import { DateTime, Effect } from "effect";
|
|
18
|
-
|
|
19
|
-
//#region src/core/builder.ts
|
|
20
|
-
const Connector = { define: (definition) => definition };
|
|
21
|
-
const Resource = {
|
|
22
|
-
entity: (definition) => definition,
|
|
23
|
-
webhook: (definition) => definition,
|
|
24
|
-
upsert: (row) => ({
|
|
25
|
-
op: "upsert",
|
|
26
|
-
row
|
|
27
|
-
}),
|
|
28
|
-
delete: (options) => ({
|
|
29
|
-
op: "delete",
|
|
30
|
-
key: options.key,
|
|
31
|
-
version: options.version
|
|
32
|
-
})
|
|
33
|
-
};
|
|
34
|
-
const Fetch = {
|
|
35
|
-
page: (definition) => definition,
|
|
36
|
-
changes: (definition) => definition
|
|
37
|
-
};
|
|
38
|
-
const decodeFailure = (kind, value) => new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });
|
|
39
|
-
const Cursor = {
|
|
40
|
-
string: () => ({
|
|
41
|
-
kind: "string",
|
|
42
|
-
decode: (value) => typeof value === "string" ? Effect.succeed(value) : Effect.fail(decodeFailure("string", value))
|
|
43
|
-
}),
|
|
44
|
-
number: () => ({
|
|
45
|
-
kind: "number",
|
|
46
|
-
decode: (value) => typeof value === "number" && Number.isFinite(value) ? Effect.succeed(value) : Effect.fail(decodeFailure("number", value))
|
|
47
|
-
}),
|
|
48
|
-
isoDateTime: () => ({
|
|
49
|
-
kind: "isoDateTime",
|
|
50
|
-
decode: (value) => {
|
|
51
|
-
if (typeof value !== "string" || Number.isNaN(Date.parse(value))) return Effect.fail(decodeFailure("ISO datetime", value));
|
|
52
|
-
return Effect.succeed(value);
|
|
53
|
-
}
|
|
54
|
-
}),
|
|
55
|
-
nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso)
|
|
56
|
-
};
|
|
57
|
-
|
|
58
|
-
//#endregion
|
|
59
|
-
export { auth_exports as Auth, Connector, connector_app_exports as ConnectorApp, ConnectorError, Cursor, Fetch, ingestion_exports as Ingestion, manifest_exports as Manifest, metrics_exports as Metrics, publisher_exports as Publisher, Resource, runtime_config_exports as RuntimeConfig, state_store_exports as StateStore, status_exports as Status, telemetry_exports as Telemetry, webhook_exports as Webhook };
|
|
60
|
-
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":[],"sources":["../src/core/builder.ts"],"sourcesContent":["import { DateTime, Effect } from \"effect\";\n\nimport type {\n ChangesFetch,\n ConnectorDefinition,\n Cursor as CursorType,\n DeleteValue,\n PageFetch,\n ResourceDefinition,\n ResourceMutation,\n ResourceSchema,\n WebhookHandler,\n} from \"./types\";\n\nimport { ConnectorError } from \"../errors\";\n\nexport const Connector = {\n define: <const Resources extends ReadonlyArray<ResourceDefinition>>(\n definition: ConnectorDefinition<Resources>,\n ) => definition,\n};\n\nexport const Resource = {\n entity: <\n S extends ResourceSchema,\n Payload = never,\n R = never,\n const Name extends string = string,\n >(\n definition: ResourceDefinition<S, Payload, R, Name>,\n ): ResourceDefinition<S, Payload, R, Name> => definition,\n\n webhook: <Payload, Row extends object = never>(definition: WebhookHandler<Row, Payload>) =>\n definition,\n\n upsert: <Row extends object>(row: Row): ResourceMutation<Row> => ({ op: \"upsert\", row }),\n\n delete: (options: {\n readonly key: DeleteValue;\n readonly version: DeleteValue;\n }): ResourceMutation<never> => ({\n op: \"delete\",\n key: options.key,\n version: options.version,\n }),\n};\n\nexport const Fetch = {\n page: <Row extends object = never, R = never>(definition: PageFetch<Row, R>) => definition,\n changes: <Row extends object = never, R = never>(definition: ChangesFetch<Row, R>) => definition,\n};\n\nconst decodeFailure = (kind: string, value: unknown) =>\n new ConnectorError({ message: `Invalid ${kind} cursor value: ${String(value)}` });\n\nexport const Cursor = {\n string: (): CursorType.Definition<string> => ({\n kind: \"string\",\n decode: (value) =>\n typeof value === \"string\"\n ? Effect.succeed(value)\n : Effect.fail(decodeFailure(\"string\", value)),\n }),\n\n number: (): CursorType.Definition<number> => ({\n kind: \"number\",\n decode: (value) =>\n typeof value === \"number\" && Number.isFinite(value)\n ? Effect.succeed(value)\n : Effect.fail(decodeFailure(\"number\", value)),\n }),\n\n isoDateTime: (): CursorType.Definition<string> => ({\n kind: \"isoDateTime\",\n decode: (value) => {\n if (typeof value !== \"string\" || Number.isNaN(Date.parse(value))) {\n return Effect.fail(decodeFailure(\"ISO datetime\", value));\n }\n return Effect.succeed(value);\n },\n }),\n\n nowIsoDateTime: Effect.map(DateTime.now, DateTime.formatIso),\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAgBA,MAAa,YAAY,EACvB,SACE,eACG,YACN;AAED,MAAa,WAAW;CACtB,SAME,eAC4C;CAE9C,UAA+C,eAC7C;CAEF,SAA6B,SAAqC;EAAE,IAAI;EAAU;EAAK;CAEvF,SAAS,aAGuB;EAC9B,IAAI;EACJ,KAAK,QAAQ;EACb,SAAS,QAAQ;EAClB;CACF;AAED,MAAa,QAAQ;CACnB,OAA8C,eAAkC;CAChF,UAAiD,eAAqC;CACvF;AAED,MAAM,iBAAiB,MAAc,UACnC,IAAI,eAAe,EAAE,SAAS,WAAW,KAAK,iBAAiB,OAAO,MAAM,IAAI,CAAC;AAEnF,MAAa,SAAS;CACpB,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,WACb,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,eAA8C;EAC5C,MAAM;EACN,SAAS,UACP,OAAO,UAAU,YAAY,OAAO,SAAS,MAAM,GAC/C,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,cAAc,UAAU,MAAM,CAAC;EAClD;CAED,oBAAmD;EACjD,MAAM;EACN,SAAS,UAAU;AACjB,OAAI,OAAO,UAAU,YAAY,OAAO,MAAM,KAAK,MAAM,MAAM,CAAC,CAC9D,QAAO,OAAO,KAAK,cAAc,gBAAgB,MAAM,CAAC;AAE1D,UAAO,OAAO,QAAQ,MAAM;;EAE/B;CAED,gBAAgB,OAAO,IAAI,SAAS,KAAK,SAAS,UAAU;CAC7D"}
|
package/dist/ingestion/index.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import "../errors-DHWylqRZ.js";
|
|
2
|
-
import "../constants-D113Ut1x.js";
|
|
3
|
-
import "../telemetry-j_nxwYfh.js";
|
|
4
|
-
import "../metrics-inkCAxKq.js";
|
|
5
|
-
import "../service-DiP-XETo.js";
|
|
6
|
-
import "../server-D-eLX3yN.js";
|
|
7
|
-
import "../state-store-BdYulQOY.js";
|
|
8
|
-
import "../status-BTTd7AUR.js";
|
|
9
|
-
import { n as run } from "../ingestion-QBGKrFr9.js";
|
|
10
|
-
|
|
11
|
-
export { run };
|
|
@@ -1,200 +0,0 @@
|
|
|
1
|
-
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
-
import { i as SpanName, o as annotateError, t as Attr } from "./telemetry-j_nxwYfh.js";
|
|
3
|
-
import { d as setLastSuccessTimestamp, f as setSyncState, p as setWebhookQueueDepth } from "./metrics-inkCAxKq.js";
|
|
4
|
-
import { n as router, r as publishBatch, t as WebhookQueue } from "./server-D-eLX3yN.js";
|
|
5
|
-
import { o as StateStore } from "./state-store-BdYulQOY.js";
|
|
6
|
-
import { c as deriveSyncState, l as normalizeCursor, s as statusRoute } from "./status-BTTd7AUR.js";
|
|
7
|
-
import { DateTime, Effect, Layer, Queue } from "effect";
|
|
8
|
-
import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
|
|
9
|
-
import * as Observability from "effect/unstable/observability";
|
|
10
|
-
|
|
11
|
-
//#region src/ingestion/engine.ts
|
|
12
|
-
function run(connector, options) {
|
|
13
|
-
const queueLayer = makeWebhookQueueLayer(connector.name);
|
|
14
|
-
const runtimeLayer = options?.webhook ? Layer.mergeAll(queueLayer, makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer))) : Layer.empty;
|
|
15
|
-
return Effect.gen(function* () {
|
|
16
|
-
const initialCutoff = options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));
|
|
17
|
-
yield* Effect.logInfo("Connector started").pipe(Effect.annotateLogs({
|
|
18
|
-
[Attr.connectorName]: connector.name,
|
|
19
|
-
resources: connector.resources.length
|
|
20
|
-
}));
|
|
21
|
-
const sourceRun = runIngestion(connector, initialCutoff);
|
|
22
|
-
const webhookRun = options?.webhook ? runWebhookRuntime(connector, options.webhook.routes) : Effect.void;
|
|
23
|
-
return yield* Effect.all([sourceRun, webhookRun], { concurrency: "unbounded" }).pipe(Effect.asVoid);
|
|
24
|
-
}).pipe(Effect.provide(runtimeLayer));
|
|
25
|
-
}
|
|
26
|
-
const makeWebhookQueueLayer = (connector) => Layer.effect(WebhookQueue)(Effect.gen(function* () {
|
|
27
|
-
const queue = yield* Queue.bounded(1024);
|
|
28
|
-
yield* setWebhookQueueDepth(connector, 0);
|
|
29
|
-
return WebhookQueue.of({ queue });
|
|
30
|
-
}));
|
|
31
|
-
const runWebhookRuntime = (connector, routes) => {
|
|
32
|
-
const hasAfterEnqueue = routes.some((route) => route.ackMode === "after-enqueue");
|
|
33
|
-
return Effect.all([hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never], { concurrency: "unbounded" }).pipe(Effect.asVoid);
|
|
34
|
-
};
|
|
35
|
-
const runWebhookQueueConsumer = (connector) => Effect.gen(function* () {
|
|
36
|
-
const queue = yield* WebhookQueue;
|
|
37
|
-
while (true) {
|
|
38
|
-
const batch = yield* Queue.take(queue.queue);
|
|
39
|
-
const depth = yield* Queue.size(queue.queue);
|
|
40
|
-
yield* setWebhookQueueDepth(connector.name, depth);
|
|
41
|
-
yield* publishBatch({
|
|
42
|
-
connector: connector.name,
|
|
43
|
-
resource: batch.resource,
|
|
44
|
-
source: "webhook",
|
|
45
|
-
batch: batch.batch
|
|
46
|
-
});
|
|
47
|
-
}
|
|
48
|
-
});
|
|
49
|
-
const makeWebhookServerLayer = (connector, options) => {
|
|
50
|
-
const healthPath = options.healthPath ?? "/health";
|
|
51
|
-
const metricsPath = options.metricsPath === void 0 ? "/metrics" : options.metricsPath;
|
|
52
|
-
const statusPath = options.statusPath === void 0 ? "/status" : options.statusPath;
|
|
53
|
-
const app = Layer.mergeAll(router(options.routes, { connectorName: connector.name }), HttpRouter.add("GET", healthPath, Effect.succeed(HttpServerResponse.text("ok"))), ...metricsPath === false ? [] : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })], ...statusPath === false ? [] : [statusRoute(connector, statusPath)]);
|
|
54
|
-
return HttpRouter.serve(app, { disableLogger: options.disableHttpLogger ?? true });
|
|
55
|
-
};
|
|
56
|
-
const runIngestion = (connector, initialCutoff) => Effect.forEach(connector.resources, (resource) => runResourceSources(connector, resource, initialCutoff), { concurrency: "unbounded" }).pipe(Effect.asVoid);
|
|
57
|
-
const initializeResourceState = (existing, initialCutoff) => ({
|
|
58
|
-
...existing,
|
|
59
|
-
changes: existing?.changes ?? { cursor: initialCutoff },
|
|
60
|
-
backfill: existing?.backfill ?? {
|
|
61
|
-
cutoff: initialCutoff,
|
|
62
|
-
completed: false
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
const getInitializedState = Effect.fnUntraced(function* (resource, initialCutoff) {
|
|
66
|
-
return initializeResourceState(yield* (yield* StateStore).getResourceState(resource), initialCutoff);
|
|
67
|
-
});
|
|
68
|
-
const runResourceSources = (connector, resource, initialCutoff) => {
|
|
69
|
-
const runs = [resource.backfill ? isolateSourceFailure(runBackfill(connector, resource, initialCutoff), connector, resource, "backfill") : Effect.void, resource.changes ? isolateSourceFailure(runChanges(connector, resource, initialCutoff), connector, resource, "changes") : Effect.void];
|
|
70
|
-
return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(Effect.andThen(Effect.all(runs, { concurrency: "unbounded" })), Effect.asVoid, Effect.catch(() => setSyncState({
|
|
71
|
-
connector: connector.name,
|
|
72
|
-
resource: resource.name
|
|
73
|
-
}, "error").pipe(Effect.andThen(Effect.logError("Connector resource initialization failed")), Effect.annotateLogs({
|
|
74
|
-
[Attr.connectorName]: connector.name,
|
|
75
|
-
resource: resource.name
|
|
76
|
-
}), Effect.andThen(Effect.never))));
|
|
77
|
-
};
|
|
78
|
-
const recordResourceError = (connector, resource, source, operation) => (effect) => effect.pipe(Effect.tapError(() => setSyncState({
|
|
79
|
-
connector: connector.name,
|
|
80
|
-
resource: resource.name
|
|
81
|
-
}, "error").pipe(Effect.andThen(StateStore.pipe(Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)))))));
|
|
82
|
-
const isolateSourceFailure = (effect, connector, resource, source) => effect.pipe(Effect.catch(() => Effect.logError("Connector source parked").pipe(Effect.annotateLogs({
|
|
83
|
-
[Attr.connectorName]: connector.name,
|
|
84
|
-
resource: resource.name,
|
|
85
|
-
source
|
|
86
|
-
}), Effect.andThen(Effect.never))));
|
|
87
|
-
const refreshRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
|
|
88
|
-
const state = yield* getInitializedState(resource.name, initialCutoff);
|
|
89
|
-
yield* setSyncState({
|
|
90
|
-
connector: connector.name,
|
|
91
|
-
resource: resource.name
|
|
92
|
-
}, deriveSyncState(resource, state));
|
|
93
|
-
return state;
|
|
94
|
-
});
|
|
95
|
-
const initializeRuntimeStatus = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
|
|
96
|
-
const state = yield* getInitializedState(resource.name, initialCutoff);
|
|
97
|
-
yield* setSyncState({
|
|
98
|
-
connector: connector.name,
|
|
99
|
-
resource: resource.name
|
|
100
|
-
}, deriveSyncState(resource, state));
|
|
101
|
-
if (resource.backfill) yield* setLastSuccessTimestamp({
|
|
102
|
-
connector: connector.name,
|
|
103
|
-
resource: resource.name,
|
|
104
|
-
source: "backfill"
|
|
105
|
-
}, state.backfill?.lastSuccessAt);
|
|
106
|
-
if (resource.changes) yield* setLastSuccessTimestamp({
|
|
107
|
-
connector: connector.name,
|
|
108
|
-
resource: resource.name,
|
|
109
|
-
source: "changes"
|
|
110
|
-
}, state.changes?.lastSuccessAt);
|
|
111
|
-
});
|
|
112
|
-
const runBackfill = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
|
|
113
|
-
if (!resource.backfill) return;
|
|
114
|
-
const store = yield* StateStore;
|
|
115
|
-
let state = yield* getInitializedState(resource.name, initialCutoff);
|
|
116
|
-
if (state.backfill?.completed === true) {
|
|
117
|
-
yield* store.clearResourceError(resource.name, "backfill").pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
|
|
118
|
-
yield* refreshRuntimeStatus(connector, resource, initialCutoff);
|
|
119
|
-
return;
|
|
120
|
-
}
|
|
121
|
-
while (state.backfill?.completed !== true) {
|
|
122
|
-
const backfill = state.backfill ?? {
|
|
123
|
-
cutoff: initialCutoff,
|
|
124
|
-
completed: false
|
|
125
|
-
};
|
|
126
|
-
const clearsVisibleError = state.lastError?.source === "backfill";
|
|
127
|
-
const page = yield* resource.backfill.fetch({
|
|
128
|
-
pageCursor: backfill.pageCursor,
|
|
129
|
-
cutoff: backfill.cutoff
|
|
130
|
-
}).pipe(recordResourceError(connector, resource, "backfill", "fetch"));
|
|
131
|
-
yield* publishBatch({
|
|
132
|
-
connector: connector.name,
|
|
133
|
-
resource: resource.name,
|
|
134
|
-
source: "backfill",
|
|
135
|
-
batch: {
|
|
136
|
-
cursor: page.nextPageCursor ?? backfill.cutoff,
|
|
137
|
-
mutations: page.mutations
|
|
138
|
-
}
|
|
139
|
-
}).pipe(recordResourceError(connector, resource, "backfill", "publish"));
|
|
140
|
-
const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
|
|
141
|
-
state = {
|
|
142
|
-
...state,
|
|
143
|
-
backfill: {
|
|
144
|
-
cutoff: backfill.cutoff,
|
|
145
|
-
pageCursor: page.nextPageCursor,
|
|
146
|
-
completed: !page.hasMore,
|
|
147
|
-
lastSuccessAt
|
|
148
|
-
}
|
|
149
|
-
};
|
|
150
|
-
yield* Effect.withSpan(store.setBackfillState(resource.name, {
|
|
151
|
-
cutoff: normalizeCursor(backfill.cutoff),
|
|
152
|
-
pageCursor: page.nextPageCursor === void 0 ? void 0 : normalizeCursor(page.nextPageCursor),
|
|
153
|
-
completed: !page.hasMore,
|
|
154
|
-
lastSuccessAt
|
|
155
|
-
}).pipe(Effect.andThen(setLastSuccessTimestamp({
|
|
156
|
-
connector: connector.name,
|
|
157
|
-
resource: resource.name,
|
|
158
|
-
source: "backfill"
|
|
159
|
-
}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "backfill")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "backfill", "checkpoint"));
|
|
160
|
-
if (clearsVisibleError || !page.hasMore) state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);
|
|
161
|
-
}
|
|
162
|
-
});
|
|
163
|
-
const runChanges = Effect.fnUntraced(function* (connector, resource, initialCutoff) {
|
|
164
|
-
if (!resource.changes) return;
|
|
165
|
-
const store = yield* StateStore;
|
|
166
|
-
while (true) {
|
|
167
|
-
const state = yield* getInitializedState(resource.name, initialCutoff);
|
|
168
|
-
const cursor = state.changes?.cursor ?? initialCutoff;
|
|
169
|
-
const clearsVisibleError = state.lastError?.source === "changes";
|
|
170
|
-
const page = yield* resource.changes.fetch({ cursor }).pipe(recordResourceError(connector, resource, "changes", "fetch"));
|
|
171
|
-
yield* publishBatch({
|
|
172
|
-
connector: connector.name,
|
|
173
|
-
resource: resource.name,
|
|
174
|
-
source: "changes",
|
|
175
|
-
batch: {
|
|
176
|
-
cursor: page.cursor,
|
|
177
|
-
mutations: page.mutations
|
|
178
|
-
}
|
|
179
|
-
}).pipe(recordResourceError(connector, resource, "changes", "publish"));
|
|
180
|
-
const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
|
|
181
|
-
yield* Effect.withSpan(store.setChangesState(resource.name, {
|
|
182
|
-
cursor: normalizeCursor(page.cursor),
|
|
183
|
-
lastSuccessAt
|
|
184
|
-
}).pipe(Effect.andThen(setLastSuccessTimestamp({
|
|
185
|
-
connector: connector.name,
|
|
186
|
-
resource: resource.name,
|
|
187
|
-
source: "changes"
|
|
188
|
-
}, lastSuccessAt)), Effect.andThen(store.clearResourceError(resource.name, "changes")), Effect.tapError((error) => annotateError("state_set", error))), SpanName.stateSet, { attributes: { [Attr.stateKey]: resource.name } }).pipe(recordResourceError(connector, resource, "changes", "checkpoint"));
|
|
189
|
-
if (clearsVisibleError) yield* refreshRuntimeStatus(connector, resource, initialCutoff);
|
|
190
|
-
yield* Effect.sleep(resource.changes.interval ?? "1 minute");
|
|
191
|
-
}
|
|
192
|
-
});
|
|
193
|
-
|
|
194
|
-
//#endregion
|
|
195
|
-
//#region src/ingestion/index.ts
|
|
196
|
-
var ingestion_exports = /* @__PURE__ */ __export({ run: () => run });
|
|
197
|
-
|
|
198
|
-
//#endregion
|
|
199
|
-
export { run as n, ingestion_exports as t };
|
|
200
|
-
//# sourceMappingURL=ingestion-QBGKrFr9.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion-QBGKrFr9.js","names":["Metrics.setWebhookQueueDepth","healthPath: HttpRouter.PathInput","Metrics.setSyncState","Metrics.setLastSuccessTimestamp"],"sources":["../src/ingestion/engine.ts","../src/ingestion/index.ts"],"sourcesContent":["import { DateTime, Effect, Layer, Queue } from \"effect\";\nimport { HttpRouter, type HttpServer, HttpServerResponse } from \"effect/unstable/http\";\nimport * as Observability from \"effect/unstable/observability\";\n\nimport type {\n ConnectorDefinition,\n Cursor,\n ResourceDefinition,\n ResourceState,\n WebhookRoute,\n} from \"../core/types\";\nimport type { StateOperation, StateSource } from \"../state-store/schema\";\n\nimport { ConnectorError } from \"../errors\";\nimport * as Metrics from \"../metrics\";\nimport { publishBatch } from \"../publisher/instrumented\";\nimport { Publisher } from \"../publisher/service\";\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, normalizeCursor } from \"../state-store/state\";\nimport { statusRoute } from \"../status\";\nimport { Attr, SpanName, annotateError } from \"../telemetry\";\nimport { router, WebhookQueue, type QueuedWebhookBatch } from \"../webhook/server\";\n\ntype RunBaseOptions = {\n readonly initialCutoff?: Cursor.Value;\n};\n\nexport type RunOptions = RunBaseOptions & {\n readonly webhook?: {\n readonly routes: ReadonlyArray<WebhookRoute>;\n readonly healthPath?: HttpRouter.PathInput;\n readonly metricsPath?: HttpRouter.PathInput | false;\n readonly statusPath?: HttpRouter.PathInput | false;\n readonly disableHttpLogger?: boolean;\n };\n};\n\ntype RunNoWebhookOptions = RunBaseOptions & {\n readonly webhook?: undefined;\n};\n\ntype RunWebhookOptions = RunOptions & {\n readonly webhook: NonNullable<RunOptions[\"webhook\"]>;\n};\n\n/**\n * Runs all connector ingestion loops. Callers provide the Publisher and\n * StateStore layers; webhook mode additionally requires an HttpServer layer.\n */\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n connector: ConnectorDefinition<Resources>,\n options?: RunNoWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher>;\nexport function run<const Resources extends ReadonlyArray<ResourceDefinition>>(\n connector: ConnectorDefinition<Resources>,\n options: RunWebhookOptions,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher | HttpServer.HttpServer>;\nexport function run(connector: ConnectorDefinition, options?: RunOptions) {\n const queueLayer = makeWebhookQueueLayer(connector.name);\n const runtimeLayer = options?.webhook\n ? Layer.mergeAll(\n queueLayer,\n makeWebhookServerLayer(connector, options.webhook).pipe(Layer.provide(queueLayer)),\n )\n : Layer.empty;\n\n return Effect.gen(function* () {\n const initialCutoff =\n options?.initialCutoff ?? (yield* DateTime.now.pipe(Effect.map(DateTime.formatIso)));\n yield* Effect.logInfo(\"Connector started\").pipe(\n Effect.annotateLogs({\n [Attr.connectorName]: connector.name,\n resources: connector.resources.length,\n }),\n );\n\n const sourceRun = runIngestion(connector, initialCutoff);\n const webhookRun = options?.webhook\n ? runWebhookRuntime(connector, options.webhook.routes)\n : Effect.void;\n\n return yield* Effect.all([sourceRun, webhookRun], { concurrency: \"unbounded\" }).pipe(\n Effect.asVoid,\n );\n }).pipe(Effect.provide(runtimeLayer));\n}\n\nconst makeWebhookQueueLayer = (connector: string): Layer.Layer<WebhookQueue> =>\n Layer.effect(WebhookQueue)(\n Effect.gen(function* () {\n const queue = yield* Queue.bounded<QueuedWebhookBatch>(1024);\n yield* Metrics.setWebhookQueueDepth(connector, 0);\n return WebhookQueue.of({ queue });\n }),\n );\n\nconst runWebhookRuntime = (connector: ConnectorDefinition, routes: ReadonlyArray<WebhookRoute>) => {\n const hasAfterEnqueue = routes.some((route) => route.ackMode === \"after-enqueue\");\n return Effect.all(\n [hasAfterEnqueue ? runWebhookQueueConsumer(connector) : Effect.void, Effect.never],\n {\n concurrency: \"unbounded\",\n },\n ).pipe(Effect.asVoid);\n};\n\nconst runWebhookQueueConsumer = (connector: ConnectorDefinition) =>\n Effect.gen(function* () {\n const queue = yield* WebhookQueue;\n while (true) {\n const batch = yield* Queue.take(queue.queue);\n const depth = yield* Queue.size(queue.queue);\n yield* Metrics.setWebhookQueueDepth(connector.name, depth);\n yield* publishBatch({\n connector: connector.name,\n resource: batch.resource,\n source: \"webhook\",\n batch: batch.batch,\n });\n }\n });\n\nconst makeWebhookServerLayer = (\n connector: ConnectorDefinition,\n options: {\n readonly routes: ReadonlyArray<WebhookRoute>;\n readonly healthPath?: HttpRouter.PathInput;\n readonly metricsPath?: HttpRouter.PathInput | false;\n readonly statusPath?: HttpRouter.PathInput | false;\n readonly disableHttpLogger?: boolean;\n },\n): Layer.Layer<never, never, HttpServer.HttpServer> => {\n const healthPath: HttpRouter.PathInput = options.healthPath ?? \"/health\";\n const metricsPath = options.metricsPath === undefined ? \"/metrics\" : options.metricsPath;\n const statusPath = options.statusPath === undefined ? \"/status\" : options.statusPath;\n const app = Layer.mergeAll(\n router(options.routes, { connectorName: connector.name }),\n HttpRouter.add(\"GET\", healthPath, Effect.succeed(HttpServerResponse.text(\"ok\"))),\n ...(metricsPath === false\n ? []\n : [Observability.PrometheusMetrics.layerHttp({ path: metricsPath })]),\n ...(statusPath === false ? [] : [statusRoute(connector, statusPath)]),\n );\n\n return HttpRouter.serve(app, {\n disableLogger: options.disableHttpLogger ?? true,\n });\n};\n\nconst runIngestion = (\n connector: ConnectorDefinition,\n initialCutoff: Cursor.Value,\n): Effect.Effect<void, ConnectorError, StateStore | Publisher> =>\n Effect.forEach(\n connector.resources,\n (resource) => runResourceSources(connector, resource, initialCutoff),\n {\n concurrency: \"unbounded\",\n },\n ).pipe(Effect.asVoid);\n\nconst initializeResourceState = (\n existing: ResourceState | undefined,\n initialCutoff: Cursor.Value,\n): ResourceState => ({\n ...existing,\n changes: existing?.changes ?? { cursor: initialCutoff },\n backfill: existing?.backfill ?? {\n cutoff: initialCutoff,\n completed: false,\n },\n});\n\nconst getInitializedState = Effect.fnUntraced(function* (\n resource: string,\n initialCutoff: Cursor.Value,\n) {\n const store = yield* StateStore;\n const existing = yield* store.getResourceState(resource);\n return initializeResourceState(existing, initialCutoff);\n});\n\nconst runResourceSources = (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n initialCutoff: Cursor.Value,\n) => {\n const runs = [\n resource.backfill\n ? isolateSourceFailure(\n runBackfill(connector, resource, initialCutoff),\n connector,\n resource,\n \"backfill\",\n )\n : Effect.void,\n resource.changes\n ? isolateSourceFailure(\n runChanges(connector, resource, initialCutoff),\n connector,\n resource,\n \"changes\",\n )\n : Effect.void,\n ];\n\n return initializeRuntimeStatus(connector, resource, initialCutoff).pipe(\n Effect.andThen(Effect.all(runs, { concurrency: \"unbounded\" })),\n Effect.asVoid,\n Effect.catch(() =>\n Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n Effect.andThen(Effect.logError(\"Connector resource initialization failed\")),\n Effect.annotateLogs({\n [Attr.connectorName]: connector.name,\n resource: resource.name,\n }),\n Effect.andThen(Effect.never),\n ),\n ),\n );\n};\n\nconst recordResourceError =\n (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n source: StateSource,\n operation: StateOperation,\n ) =>\n <A, R>(\n effect: Effect.Effect<A, ConnectorError, R>,\n ): Effect.Effect<A, ConnectorError, R | StateStore> =>\n effect.pipe(\n Effect.tapError(() =>\n Metrics.setSyncState({ connector: connector.name, resource: resource.name }, \"error\").pipe(\n Effect.andThen(\n StateStore.pipe(\n Effect.flatMap((store) => store.setResourceError(resource.name, source, operation)),\n ),\n ),\n ),\n ),\n );\n\nconst isolateSourceFailure = <A, R>(\n effect: Effect.Effect<A, ConnectorError, R>,\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n source: StateSource,\n): Effect.Effect<A, never, R> =>\n // Only typed source errors are isolated. Defects and interruption still stop the runtime.\n effect.pipe(\n Effect.catch(() =>\n Effect.logError(\"Connector source parked\").pipe(\n Effect.annotateLogs({\n [Attr.connectorName]: connector.name,\n resource: resource.name,\n source,\n }),\n Effect.andThen(Effect.never),\n ),\n ),\n );\n\nconst refreshRuntimeStatus = Effect.fnUntraced(function* (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n initialCutoff: Cursor.Value,\n) {\n const state = yield* getInitializedState(resource.name, initialCutoff);\n yield* Metrics.setSyncState(\n { connector: connector.name, resource: resource.name },\n deriveSyncState(resource, state),\n );\n return state;\n});\n\nconst initializeRuntimeStatus = Effect.fnUntraced(function* (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n initialCutoff: Cursor.Value,\n) {\n const state = yield* getInitializedState(resource.name, initialCutoff);\n yield* Metrics.setSyncState(\n { connector: connector.name, resource: resource.name },\n deriveSyncState(resource, state),\n );\n if (resource.backfill) {\n yield* Metrics.setLastSuccessTimestamp(\n { connector: connector.name, resource: resource.name, source: \"backfill\" },\n state.backfill?.lastSuccessAt,\n );\n }\n if (resource.changes) {\n yield* Metrics.setLastSuccessTimestamp(\n { connector: connector.name, resource: resource.name, source: \"changes\" },\n state.changes?.lastSuccessAt,\n );\n }\n});\n\nconst runBackfill = Effect.fnUntraced(function* (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n initialCutoff: Cursor.Value,\n) {\n if (!resource.backfill) return;\n const store = yield* StateStore;\n let state = yield* getInitializedState(resource.name, initialCutoff);\n\n if (state.backfill?.completed === true) {\n // The checkpoint may have committed before the previous process cleared\n // its error marker. It may also be hidden by a newer changes error.\n yield* store\n .clearResourceError(resource.name, \"backfill\")\n .pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n return;\n }\n\n while (state.backfill?.completed !== true) {\n const backfill = state.backfill ?? { cutoff: initialCutoff, completed: false };\n const clearsVisibleError = state.lastError?.source === \"backfill\";\n const page = yield* resource.backfill\n .fetch({\n pageCursor: backfill.pageCursor,\n cutoff: backfill.cutoff,\n })\n .pipe(recordResourceError(connector, resource, \"backfill\", \"fetch\"));\n\n yield* publishBatch({\n connector: connector.name,\n resource: resource.name,\n source: \"backfill\",\n batch: {\n cursor: page.nextPageCursor ?? backfill.cutoff,\n mutations: page.mutations,\n },\n }).pipe(recordResourceError(connector, resource, \"backfill\", \"publish\"));\n\n const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n state = {\n ...state,\n backfill: {\n cutoff: backfill.cutoff,\n pageCursor: page.nextPageCursor,\n completed: !page.hasMore,\n lastSuccessAt,\n },\n };\n\n // Checkpoint only after publish succeeds so a crash cannot skip an\n // unacknowledged page; replay therefore remains at-least-once.\n yield* Effect.withSpan(\n store\n .setBackfillState(resource.name, {\n cutoff: normalizeCursor(backfill.cutoff),\n pageCursor:\n page.nextPageCursor === undefined ? undefined : normalizeCursor(page.nextPageCursor),\n completed: !page.hasMore,\n lastSuccessAt,\n })\n .pipe(\n Effect.andThen(\n Metrics.setLastSuccessTimestamp(\n { connector: connector.name, resource: resource.name, source: \"backfill\" },\n lastSuccessAt,\n ),\n ),\n Effect.andThen(store.clearResourceError(resource.name, \"backfill\")),\n Effect.tapError((error) => annotateError(\"state_set\", error)),\n ),\n SpanName.stateSet,\n { attributes: { [Attr.stateKey]: resource.name } },\n ).pipe(recordResourceError(connector, resource, \"backfill\", \"checkpoint\"));\n\n // Completion or clearing an error can change the resource-wide sync state.\n if (clearsVisibleError || !page.hasMore) {\n state = yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n }\n }\n});\n\nconst runChanges = Effect.fnUntraced(function* (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n initialCutoff: Cursor.Value,\n) {\n if (!resource.changes) return;\n const store = yield* StateStore;\n\n while (true) {\n const state = yield* getInitializedState(resource.name, initialCutoff);\n const cursor = state.changes?.cursor ?? initialCutoff;\n const clearsVisibleError = state.lastError?.source === \"changes\";\n const page = yield* resource.changes\n .fetch({ cursor })\n .pipe(recordResourceError(connector, resource, \"changes\", \"fetch\"));\n\n yield* publishBatch({\n connector: connector.name,\n resource: resource.name,\n source: \"changes\",\n batch: {\n cursor: page.cursor,\n mutations: page.mutations,\n },\n }).pipe(recordResourceError(connector, resource, \"changes\", \"publish\"));\n\n const lastSuccessAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));\n // Advance the durable cursor only after Wings accepts the page.\n yield* Effect.withSpan(\n store\n .setChangesState(resource.name, {\n cursor: normalizeCursor(page.cursor),\n lastSuccessAt,\n })\n .pipe(\n Effect.andThen(\n Metrics.setLastSuccessTimestamp(\n { connector: connector.name, resource: resource.name, source: \"changes\" },\n lastSuccessAt,\n ),\n ),\n Effect.andThen(store.clearResourceError(resource.name, \"changes\")),\n Effect.tapError((error) => annotateError(\"state_set\", error)),\n ),\n SpanName.stateSet,\n { attributes: { [Attr.stateKey]: resource.name } },\n ).pipe(recordResourceError(connector, resource, \"changes\", \"checkpoint\"));\n\n if (clearsVisibleError) {\n yield* refreshRuntimeStatus(connector, resource, initialCutoff);\n }\n\n yield* Effect.sleep(resource.changes.interval ?? \"1 minute\");\n }\n});\n","export { run, type RunOptions } from \"./engine\";\n"],"mappings":";;;;;;;;;;;AAyDA,SAAgB,IAAI,WAAgC,SAAsB;CACxE,MAAM,aAAa,sBAAsB,UAAU,KAAK;CACxD,MAAM,eAAe,SAAS,UAC1B,MAAM,SACJ,YACA,uBAAuB,WAAW,QAAQ,QAAQ,CAAC,KAAK,MAAM,QAAQ,WAAW,CAAC,CACnF,GACD,MAAM;AAEV,QAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,gBACJ,SAAS,kBAAkB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AACrF,SAAO,OAAO,QAAQ,oBAAoB,CAAC,KACzC,OAAO,aAAa;IACjB,KAAK,gBAAgB,UAAU;GAChC,WAAW,UAAU,UAAU;GAChC,CAAC,CACH;EAED,MAAM,YAAY,aAAa,WAAW,cAAc;EACxD,MAAM,aAAa,SAAS,UACxB,kBAAkB,WAAW,QAAQ,QAAQ,OAAO,GACpD,OAAO;AAEX,SAAO,OAAO,OAAO,IAAI,CAAC,WAAW,WAAW,EAAE,EAAE,aAAa,aAAa,CAAC,CAAC,KAC9E,OAAO,OACR;GACD,CAAC,KAAK,OAAO,QAAQ,aAAa,CAAC;;AAGvC,MAAM,yBAAyB,cAC7B,MAAM,OAAO,aAAa,CACxB,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,MAAM,QAA4B,KAAK;AAC5D,QAAOA,qBAA6B,WAAW,EAAE;AACjD,QAAO,aAAa,GAAG,EAAE,OAAO,CAAC;EACjC,CACH;AAEH,MAAM,qBAAqB,WAAgC,WAAwC;CACjG,MAAM,kBAAkB,OAAO,MAAM,UAAU,MAAM,YAAY,gBAAgB;AACjF,QAAO,OAAO,IACZ,CAAC,kBAAkB,wBAAwB,UAAU,GAAG,OAAO,MAAM,OAAO,MAAM,EAClF,EACE,aAAa,aACd,CACF,CAAC,KAAK,OAAO,OAAO;;AAGvB,MAAM,2BAA2B,cAC/B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;AACrB,QAAO,MAAM;EACX,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,MAAM;EAC5C,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,MAAM;AAC5C,SAAOA,qBAA6B,UAAU,MAAM,MAAM;AAC1D,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,MAAM;GAChB,QAAQ;GACR,OAAO,MAAM;GACd,CAAC;;EAEJ;AAEJ,MAAM,0BACJ,WACA,YAOqD;CACrD,MAAMC,aAAmC,QAAQ,cAAc;CAC/D,MAAM,cAAc,QAAQ,gBAAgB,SAAY,aAAa,QAAQ;CAC7E,MAAM,aAAa,QAAQ,eAAe,SAAY,YAAY,QAAQ;CAC1E,MAAM,MAAM,MAAM,SAChB,OAAO,QAAQ,QAAQ,EAAE,eAAe,UAAU,MAAM,CAAC,EACzD,WAAW,IAAI,OAAO,YAAY,OAAO,QAAQ,mBAAmB,KAAK,KAAK,CAAC,CAAC,EAChF,GAAI,gBAAgB,QAChB,EAAE,GACF,CAAC,cAAc,kBAAkB,UAAU,EAAE,MAAM,aAAa,CAAC,CAAC,EACtE,GAAI,eAAe,QAAQ,EAAE,GAAG,CAAC,YAAY,WAAW,WAAW,CAAC,CACrE;AAED,QAAO,WAAW,MAAM,KAAK,EAC3B,eAAe,QAAQ,qBAAqB,MAC7C,CAAC;;AAGJ,MAAM,gBACJ,WACA,kBAEA,OAAO,QACL,UAAU,YACT,aAAa,mBAAmB,WAAW,UAAU,cAAc,EACpE,EACE,aAAa,aACd,CACF,CAAC,KAAK,OAAO,OAAO;AAEvB,MAAM,2BACJ,UACA,mBACmB;CACnB,GAAG;CACH,SAAS,UAAU,WAAW,EAAE,QAAQ,eAAe;CACvD,UAAU,UAAU,YAAY;EAC9B,QAAQ;EACR,WAAW;EACZ;CACF;AAED,MAAM,sBAAsB,OAAO,WAAW,WAC5C,UACA,eACA;AAGA,QAAO,wBADU,QADH,OAAO,YACS,iBAAiB,SAAS,EACf,cAAc;EACvD;AAEF,MAAM,sBACJ,WACA,UACA,kBACG;CACH,MAAM,OAAO,CACX,SAAS,WACL,qBACE,YAAY,WAAW,UAAU,cAAc,EAC/C,WACA,UACA,WACD,GACD,OAAO,MACX,SAAS,UACL,qBACE,WAAW,WAAW,UAAU,cAAc,EAC9C,WACA,UACA,UACD,GACD,OAAO,KACZ;AAED,QAAO,wBAAwB,WAAW,UAAU,cAAc,CAAC,KACjE,OAAO,QAAQ,OAAO,IAAI,MAAM,EAAE,aAAa,aAAa,CAAC,CAAC,EAC9D,OAAO,QACP,OAAO,YACLC,aAAqB;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EAAE,QAAQ,CAAC,KACpF,OAAO,QAAQ,OAAO,SAAS,2CAA2C,CAAC,EAC3E,OAAO,aAAa;GACjB,KAAK,gBAAgB,UAAU;EAChC,UAAU,SAAS;EACpB,CAAC,EACF,OAAO,QAAQ,OAAO,MAAM,CAC7B,CACF,CACF;;AAGH,MAAM,uBAEF,WACA,UACA,QACA,eAGA,WAEA,OAAO,KACL,OAAO,eACLA,aAAqB;CAAE,WAAW,UAAU;CAAM,UAAU,SAAS;CAAM,EAAE,QAAQ,CAAC,KACpF,OAAO,QACL,WAAW,KACT,OAAO,SAAS,UAAU,MAAM,iBAAiB,SAAS,MAAM,QAAQ,UAAU,CAAC,CACpF,CACF,CACF,CACF,CACF;AAEL,MAAM,wBACJ,QACA,WACA,UACA,WAGA,OAAO,KACL,OAAO,YACL,OAAO,SAAS,0BAA0B,CAAC,KACzC,OAAO,aAAa;EACjB,KAAK,gBAAgB,UAAU;CAChC,UAAU,SAAS;CACnB;CACD,CAAC,EACF,OAAO,QAAQ,OAAO,MAAM,CAC7B,CACF,CACF;AAEH,MAAM,uBAAuB,OAAO,WAAW,WAC7C,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AACtE,QAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EACtD,gBAAgB,UAAU,MAAM,CACjC;AACD,QAAO;EACP;AAEF,MAAM,0BAA0B,OAAO,WAAW,WAChD,WACA,UACA,eACA;CACA,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AACtE,QAAOA,aACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,EACtD,gBAAgB,UAAU,MAAM,CACjC;AACD,KAAI,SAAS,SACX,QAAOC,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;EAAY,EAC1E,MAAM,UAAU,cACjB;AAEH,KAAI,SAAS,QACX,QAAOA,wBACL;EAAE,WAAW,UAAU;EAAM,UAAU,SAAS;EAAM,QAAQ;EAAW,EACzE,MAAM,SAAS,cAChB;EAEH;AAEF,MAAM,cAAc,OAAO,WAAW,WACpC,WACA,UACA,eACA;AACA,KAAI,CAAC,SAAS,SAAU;CACxB,MAAM,QAAQ,OAAO;CACrB,IAAI,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;AAEpE,KAAI,MAAM,UAAU,cAAc,MAAM;AAGtC,SAAO,MACJ,mBAAmB,SAAS,MAAM,WAAW,CAC7C,KAAK,oBAAoB,WAAW,UAAU,YAAY,aAAa,CAAC;AAC3E,SAAO,qBAAqB,WAAW,UAAU,cAAc;AAC/D;;AAGF,QAAO,MAAM,UAAU,cAAc,MAAM;EACzC,MAAM,WAAW,MAAM,YAAY;GAAE,QAAQ;GAAe,WAAW;GAAO;EAC9E,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,SAC1B,MAAM;GACL,YAAY,SAAS;GACrB,QAAQ,SAAS;GAClB,CAAC,CACD,KAAK,oBAAoB,WAAW,UAAU,YAAY,QAAQ,CAAC;AAEtE,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK,kBAAkB,SAAS;IACxC,WAAW,KAAK;IACjB;GACF,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,UAAU,CAAC;EAExE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AAC9E,UAAQ;GACN,GAAG;GACH,UAAU;IACR,QAAQ,SAAS;IACjB,YAAY,KAAK;IACjB,WAAW,CAAC,KAAK;IACjB;IACD;GACF;AAID,SAAO,OAAO,SACZ,MACG,iBAAiB,SAAS,MAAM;GAC/B,QAAQ,gBAAgB,SAAS,OAAO;GACxC,YACE,KAAK,mBAAmB,SAAY,SAAY,gBAAgB,KAAK,eAAe;GACtF,WAAW,CAAC,KAAK;GACjB;GACD,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;GAAY,EAC1E,cACD,CACF,EACD,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,WAAW,CAAC,EACnE,OAAO,UAAU,UAAU,cAAc,aAAa,MAAM,CAAC,CAC9D,EACH,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,MAAM,EAAE,CACnD,CAAC,KAAK,oBAAoB,WAAW,UAAU,YAAY,aAAa,CAAC;AAG1E,MAAI,sBAAsB,CAAC,KAAK,QAC9B,SAAQ,OAAO,qBAAqB,WAAW,UAAU,cAAc;;EAG3E;AAEF,MAAM,aAAa,OAAO,WAAW,WACnC,WACA,UACA,eACA;AACA,KAAI,CAAC,SAAS,QAAS;CACvB,MAAM,QAAQ,OAAO;AAErB,QAAO,MAAM;EACX,MAAM,QAAQ,OAAO,oBAAoB,SAAS,MAAM,cAAc;EACtE,MAAM,SAAS,MAAM,SAAS,UAAU;EACxC,MAAM,qBAAqB,MAAM,WAAW,WAAW;EACvD,MAAM,OAAO,OAAO,SAAS,QAC1B,MAAM,EAAE,QAAQ,CAAC,CACjB,KAAK,oBAAoB,WAAW,UAAU,WAAW,QAAQ,CAAC;AAErE,SAAO,aAAa;GAClB,WAAW,UAAU;GACrB,UAAU,SAAS;GACnB,QAAQ;GACR,OAAO;IACL,QAAQ,KAAK;IACb,WAAW,KAAK;IACjB;GACF,CAAC,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,UAAU,CAAC;EAEvE,MAAM,gBAAgB,OAAO,SAAS,IAAI,KAAK,OAAO,IAAI,SAAS,UAAU,CAAC;AAE9E,SAAO,OAAO,SACZ,MACG,gBAAgB,SAAS,MAAM;GAC9B,QAAQ,gBAAgB,KAAK,OAAO;GACpC;GACD,CAAC,CACD,KACC,OAAO,QACLA,wBACE;GAAE,WAAW,UAAU;GAAM,UAAU,SAAS;GAAM,QAAQ;GAAW,EACzE,cACD,CACF,EACD,OAAO,QAAQ,MAAM,mBAAmB,SAAS,MAAM,UAAU,CAAC,EAClE,OAAO,UAAU,UAAU,cAAc,aAAa,MAAM,CAAC,CAC9D,EACH,SAAS,UACT,EAAE,YAAY,GAAG,KAAK,WAAW,SAAS,MAAM,EAAE,CACnD,CAAC,KAAK,oBAAoB,WAAW,UAAU,WAAW,aAAa,CAAC;AAEzE,MAAI,mBACF,QAAO,qBAAqB,WAAW,UAAU,cAAc;AAGjE,SAAO,OAAO,MAAM,SAAS,QAAQ,YAAY,WAAW;;EAE9D"}
|
package/dist/manifest/index.d.ts
DELETED
|
@@ -1,2 +0,0 @@
|
|
|
1
|
-
import { _ as number, a as configSchema, b as select, c as ConnectorManifest, d as ConfigField, f as ConfigFieldSpec, g as define$1, h as boolean, i as RuntimeConfigDocument, l as ResourceCapability, m as ConnectorConfigDef, n as ConnectorConfigSchema, o as decodeConfig, p as ConfigValuesOf, r as ConnectorConfigValues, s as toRuntimeDocument, u as define, v as optional, x as string, y as secret } from "../index-CR9HJDdr.js";
|
|
2
|
-
export { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef, ConnectorConfigSchema, ConnectorConfigValues, ConnectorManifest, ResourceCapability, RuntimeConfigDocument, boolean, configSchema, decodeConfig, define, define$1 as defineConfig, number, optional, secret, select, string, toRuntimeDocument };
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"manifest-P7kBnch9.js","names":["define","document: Record<string, string | number | boolean>","value: unknown"],"sources":["../src/manifest/number-schema.ts","../src/manifest/config.ts","../src/manifest/manifest.ts","../src/manifest/schema.ts","../src/manifest/index.ts"],"sourcesContent":["import { Schema } from \"effect\";\n\nexport type NumberConstraints = {\n readonly integer?: boolean;\n readonly minimum?: number;\n};\n\nexport const makeNumberSchema = (constraints: NumberConstraints) => {\n if (constraints.integer) {\n return constraints.minimum === undefined\n ? Schema.Int\n : Schema.Int.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));\n }\n return constraints.minimum === undefined\n ? Schema.Finite\n : Schema.Finite.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));\n};\n","import { Config, Option, Redacted, Schema, Struct } from \"effect\";\n\nimport { isPlatformRuntimeKey } from \"../runtime-config/constants\";\nimport { makeNumberSchema, type NumberConstraints } from \"./number-schema\";\n\ntype ConfigFieldSpecBase = {\n readonly runtimeKey: string;\n readonly required: boolean;\n readonly description?: string;\n};\n\ntype ConfigFieldDefinition = ConfigFieldSpecBase &\n (\n | {\n readonly type: \"string\";\n readonly secret: false;\n readonly default?: string;\n readonly values?: never;\n }\n | {\n readonly type: \"string\";\n readonly secret: true;\n readonly default?: never;\n readonly values?: never;\n }\n | {\n readonly type: \"number\";\n readonly secret: false;\n readonly default?: number;\n readonly values?: never;\n readonly integer?: boolean;\n readonly minimum?: number;\n }\n | {\n readonly type: \"boolean\";\n readonly secret: false;\n readonly default?: boolean;\n readonly values?: never;\n }\n | {\n readonly type: \"select\";\n readonly secret: false;\n readonly default?: string;\n readonly values: readonly [string, ...Array<string>];\n }\n );\n\n/** Serializable, browser-safe metadata for one user-provided connector setting. */\nexport type ConfigFieldSpec = ConfigFieldDefinition & { readonly name: string };\n\n/** Pairs an Effect runtime config with the metadata used to generate its form field. */\nexport type ConfigField<A> = {\n readonly config: Config.Config<A>;\n readonly spec: ConfigFieldDefinition;\n};\n\ntype FieldBaseOptions = {\n readonly runtimeKey: string;\n readonly description?: string;\n};\n\ntype FieldOptions<A extends string | number | boolean = string | number | boolean> =\n FieldBaseOptions &\n (\n | {\n readonly required?: true;\n readonly default?: A;\n }\n | {\n readonly required: false;\n readonly default: A;\n }\n );\n\ntype NumberFieldOptions = FieldOptions<number> & NumberConstraints;\n\ntype FieldsRecord = Record<string, ConfigField<unknown>>;\n\n/** Preserves each field's value type while `Struct.map` extracts its Effect Config. */\ninterface ConfigFieldToConfig extends Struct.Lambda {\n <A>(field: ConfigField<A>): Config.Config<A>;\n readonly \"~lambda.out\": this[\"~lambda.in\"] extends ConfigField<infer A>\n ? Config.Config<A>\n : never;\n}\n\nconst configFieldToConfig = Struct.lambda<ConfigFieldToConfig>((field) => field.config);\n\ntype ConfigOf<Fields extends FieldsRecord> = {\n readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A> ? A : never;\n};\n\n/** Effect config and serializable manifest metadata produced from one field definition. */\nexport type ConnectorConfigDef<Fields extends FieldsRecord> = {\n readonly fields: {\n readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A>\n ? Config.Config<A>\n : never;\n };\n readonly config: Config.Config<ConfigOf<Fields>>;\n readonly spec: ReadonlyArray<ConfigFieldSpec>;\n};\n\nexport type ConfigValuesOf<Def extends ConnectorConfigDef<FieldsRecord>> =\n Def extends ConnectorConfigDef<infer Fields> ? ConfigOf<Fields> : never;\n\nconst withDefault = <A extends string | number | boolean>(\n config: Config.Config<A>,\n defaultValue: A | undefined,\n) => (defaultValue === undefined ? config : Config.withDefault(config, defaultValue));\n\n/** Defines a required non-empty string, optionally with a default. */\nexport const string = (options: FieldOptions<string>): ConfigField<string> => ({\n config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"string\",\n required: options.required ?? true,\n secret: false,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(options.default !== undefined ? { default: options.default } : {}),\n },\n});\n\n/** Defines a finite number with optional integer, minimum, and default constraints. */\nexport const number = (options: NumberFieldOptions): ConfigField<number> => {\n const schema = makeNumberSchema(options);\n const defaultValue =\n options.default === undefined ? undefined : Schema.decodeUnknownSync(schema)(options.default);\n\n return {\n config: withDefault(Config.schema(schema, options.runtimeKey), defaultValue),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"number\",\n required: options.required ?? true,\n secret: false,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(defaultValue !== undefined ? { default: defaultValue } : {}),\n ...(options.integer !== undefined ? { integer: options.integer } : {}),\n ...(options.minimum !== undefined ? { minimum: options.minimum } : {}),\n },\n };\n};\n\n/** Defines a boolean, optionally with a default. */\nexport const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> => ({\n config: withDefault(Config.boolean(options.runtimeKey), options.default),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"boolean\",\n required: options.required ?? true,\n secret: false,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(options.default !== undefined ? { default: options.default } : {}),\n },\n});\n\n/** Defines a string literal choice whose values are preserved in the inferred type. */\nexport const select = <const Values extends readonly [string, ...Array<string>]>(\n options: FieldOptions<Values[number]> & { readonly values: Values },\n): ConfigField<Values[number]> => ({\n config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"select\",\n required: options.required ?? true,\n secret: false,\n values: options.values,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(options.default !== undefined ? { default: options.default } : {}),\n },\n});\n\n/** Defines a required, non-empty secret decoded as `Redacted<string>`. */\nexport const secret = (\n options: FieldBaseOptions & { readonly required?: true },\n): ConfigField<Redacted.Redacted<string>> => ({\n config: Config.schema(Schema.Redacted(Schema.NonEmptyString), options.runtimeKey),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"string\",\n required: options.required ?? true,\n secret: true,\n ...(options.description !== undefined ? { description: options.description } : {}),\n },\n});\n\n/** Makes an existing field optional and represents a missing value as `Option.none()`. */\nexport const optional = <A>(field: ConfigField<A>): ConfigField<Option.Option<A>> => ({\n config: Config.option(field.config),\n spec: {\n ...field.spec,\n required: false,\n },\n});\n\n/**\n * Defines user-facing connector config once for both Effect runtime loading and\n * frontend form generation. The object keys become logical form/API field names;\n * `runtimeKey` remains the key read by Effect Config.\n *\n * @example\n * ```ts\n * const config = define({\n * apiUrl: string({ runtimeKey: \"API_URL\" }),\n * token: secret({ runtimeKey: \"API_TOKEN\" }),\n * });\n *\n * const RuntimeConfig = config.config;\n * const formFields = config.spec;\n * ```\n */\nexport const define = <const Fields extends FieldsRecord>(\n fields: Fields,\n): ConnectorConfigDef<Fields> => {\n const spec = Object.entries(fields).map(([name, field]) => ({ name, ...field.spec }));\n const platformField = spec.find((field) => isPlatformRuntimeKey(field.runtimeKey));\n if (platformField !== undefined) {\n throw new Error(`Connector runtime key is platform-owned: ${platformField.runtimeKey}`);\n }\n\n const configFields = Struct.map(fields, configFieldToConfig);\n\n return {\n fields: configFields,\n // Effect's recursive Config.Wrap cannot reduce this generic mapped type,\n // although ConfigFieldToConfig preserves the value type of every property.\n config: Config.unwrap(configFields as Config.Wrap<ConfigOf<Fields>>),\n spec,\n };\n};\n","import type { ConfigFieldSpec } from \"./config\";\n\nexport type ResourceCapability = \"backfill\" | \"changes\" | \"webhook\";\n\n/** Browser-safe connector metadata consumed by the platform and runtime adapters. */\nexport type ConnectorManifest = {\n readonly name: string;\n readonly title: string;\n readonly description?: string;\n readonly config: ReadonlyArray<ConfigFieldSpec>;\n readonly resources: ReadonlyArray<{\n readonly name: string;\n readonly capabilities: ReadonlyArray<ResourceCapability>;\n }>;\n};\n\n/**\n * Defines browser-safe connector metadata while preserving literal names and capabilities.\n *\n * @example\n * ```ts\n * const manifest = define({\n * name: \"producer-example\",\n * title: \"Example\",\n * config: ExampleConfig.spec,\n * resources: [{ name: \"orders\", capabilities: [\"backfill\"] }],\n * });\n * ```\n */\nexport const define = <const Manifest extends ConnectorManifest>(manifest: Manifest): Manifest =>\n manifest;\n","import { Predicate, Schema, SchemaTransformation, type Types } from \"effect\";\n\nimport type { ConfigFieldSpec } from \"./config\";\nimport type { ConnectorManifest } from \"./manifest\";\n\nimport { makeNumberSchema } from \"./number-schema\";\n\ntype FieldValueOf<Field extends ConfigFieldSpec> = Field[\"type\"] extends \"number\"\n ? number\n : Field[\"type\"] extends \"boolean\"\n ? boolean\n : Field[\"values\"] extends ReadonlyArray<string>\n ? Field[\"values\"][number]\n : string;\n\nexport type ConnectorConfigValues<Manifest extends ConnectorManifest> = Types.Simplify<\n {\n readonly [Field in Manifest[\"config\"][number] as Field[\"required\"] extends true\n ? Field[\"name\"]\n : never]: FieldValueOf<Field>;\n } & {\n readonly [Field in Manifest[\"config\"][number] as Field[\"required\"] extends true\n ? never\n : Field[\"name\"]]?: FieldValueOf<Field> | undefined;\n }\n> &\n object;\n\nexport type ConnectorConfigSchema<Manifest extends ConnectorManifest> = Schema.Codec<\n ConnectorConfigValues<Manifest>,\n unknown\n>;\n\n/** Flat primitive document keyed by the runtime keys consumed by Effect Config. */\nexport type RuntimeConfigDocument = Readonly<Record<string, string | number | boolean>>;\n\n// Browser forms submit numbers and booleans as strings, while API callers may\n// already send their canonical JSON representations. These schemas accept both.\nconst numberFromForm = (field: Extract<ConfigFieldSpec, { readonly type: \"number\" }>) => {\n const numberSchema = makeNumberSchema(field);\n return Schema.Union([\n numberSchema,\n Schema.NonEmptyString.pipe(\n Schema.decodeTo(numberSchema, SchemaTransformation.numberFromString),\n ),\n ]);\n};\n\nconst BooleanFromForm = Schema.Union([\n Schema.Boolean,\n Schema.Literals([\"true\", \"false\"]).pipe(\n Schema.decodeTo(\n Schema.Boolean,\n SchemaTransformation.transform({\n decode: (value) => value === \"true\",\n encode: (value) => (value ? \"true\" : \"false\"),\n }),\n ),\n ),\n]);\n\n// An empty optional form control means \"not supplied\", not an empty config value.\nconst EmptyStringAsUndefined = Schema.Literal(\"\").pipe(\n Schema.decodeTo(\n Schema.Undefined,\n SchemaTransformation.transform({ decode: () => undefined, encode: () => \"\" }),\n ),\n);\n\nconst baseFieldSchema = (field: ConfigFieldSpec): Schema.Codec<unknown, unknown> => {\n switch (field.type) {\n case \"number\":\n return numberFromForm(field);\n case \"boolean\":\n return BooleanFromForm;\n case \"select\":\n return Schema.Literals(field.values);\n case \"string\":\n return Schema.NonEmptyString;\n }\n};\n\nconst fieldSchema = (field: ConfigFieldSpec) =>\n field.required\n ? baseFieldSchema(field)\n : Schema.optional(Schema.Union([EmptyStringAsUndefined, baseFieldSchema(field)]));\n\n/**\n * Builds an Effect Schema for logical form/API values from a connector manifest.\n *\n * @example\n * ```ts\n * const decode = Schema.decodeUnknownEffect(configSchema(manifest));\n * const values = decode({\n * apiUrl: \"https://example.com\",\n * retries: \"3\",\n * });\n * ```\n */\nexport const configSchema = <const Manifest extends ConnectorManifest>(\n manifest: Manifest,\n): ConnectorConfigSchema<Manifest> => {\n const fields = Object.fromEntries(\n manifest.config.map((field) => [field.name, fieldSchema(field)]),\n );\n // Object.fromEntries cannot retain keys discovered from the runtime manifest;\n // the public mapped type restores the same relationship for callers.\n return Schema.Struct(fields) as unknown as ConnectorConfigSchema<Manifest>;\n};\n\n/** Decodes browser/API config, applies manifest defaults, and rejects undeclared fields. */\nexport const decodeConfig = <const Manifest extends ConnectorManifest>(\n manifest: Manifest,\n input: unknown,\n) => {\n const defaults = Object.fromEntries(\n manifest.config.flatMap((field) =>\n field.default === undefined ? [] : [[field.name, field.default]],\n ),\n );\n const optionalFields = new Set(\n manifest.config.flatMap((field) => (field.required ? [] : [field.name])),\n );\n const normalizedInput = Predicate.isObject(input)\n ? Object.fromEntries(\n Object.entries(input).filter(([name, value]) => value !== \"\" || !optionalFields.has(name)),\n )\n : input;\n const inputWithDefaults = Predicate.isObject(normalizedInput)\n ? Object.assign({}, defaults, normalizedInput)\n : normalizedInput;\n\n return Schema.decodeUnknownEffect(configSchema(manifest))(inputWithDefaults, {\n onExcessProperty: \"error\",\n });\n};\n\n/**\n * Maps validated logical values to the flat runtime-key document mounted in a pod.\n * Missing optional values are omitted.\n *\n * @example\n * ```ts\n * toRuntimeDocument(manifest, { apiUrl: \"https://example.com\" });\n * // { API_URL: \"https://example.com\" }\n * ```\n */\nexport const toRuntimeDocument = <const Manifest extends ConnectorManifest>(\n manifest: Manifest,\n values: ConnectorConfigValues<Manifest>,\n): RuntimeConfigDocument => {\n const document: Record<string, string | number | boolean> = {};\n for (const field of manifest.config) {\n const value: unknown = Reflect.get(values, field.name);\n if (typeof value === \"string\" || typeof value === \"number\" || typeof value === \"boolean\") {\n document[field.runtimeKey] = value;\n }\n }\n return document;\n};\n","export {\n boolean,\n define as defineConfig,\n number,\n optional,\n secret,\n select,\n string,\n} from \"./config\";\nexport type { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef } from \"./config\";\nexport { define } from \"./manifest\";\nexport type { ConnectorManifest, ResourceCapability } from \"./manifest\";\nexport { configSchema, decodeConfig, toRuntimeDocument } from \"./schema\";\nexport type { ConnectorConfigSchema, ConnectorConfigValues, RuntimeConfigDocument } from \"./schema\";\n"],"mappings":";;;;;AAOA,MAAa,oBAAoB,gBAAmC;AAClE,KAAI,YAAY,QACd,QAAO,YAAY,YAAY,SAC3B,OAAO,MACP,OAAO,IAAI,MAAM,OAAO,uBAAuB,YAAY,QAAQ,CAAC;AAE1E,QAAO,YAAY,YAAY,SAC3B,OAAO,SACP,OAAO,OAAO,MAAM,OAAO,uBAAuB,YAAY,QAAQ,CAAC;;;;;ACuE7E,MAAM,sBAAsB,OAAO,QAA6B,UAAU,MAAM,OAAO;AAoBvF,MAAM,eACJ,QACA,iBACI,iBAAiB,SAAY,SAAS,OAAO,YAAY,QAAQ,aAAa;;AAGpF,MAAa,UAAU,aAAwD;CAC7E,QAAQ,YAAY,OAAO,eAAe,QAAQ,WAAW,EAAE,QAAQ,QAAQ;CAC/E,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACjF,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;EACtE;CACF;;AAGD,MAAa,UAAU,YAAqD;CAC1E,MAAM,SAAS,iBAAiB,QAAQ;CACxC,MAAM,eACJ,QAAQ,YAAY,SAAY,SAAY,OAAO,kBAAkB,OAAO,CAAC,QAAQ,QAAQ;AAE/F,QAAO;EACL,QAAQ,YAAY,OAAO,OAAO,QAAQ,QAAQ,WAAW,EAAE,aAAa;EAC5E,MAAM;GACJ,YAAY,QAAQ;GACpB,MAAM;GACN,UAAU,QAAQ,YAAY;GAC9B,QAAQ;GACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,GAAI,iBAAiB,SAAY,EAAE,SAAS,cAAc,GAAG,EAAE;GAC/D,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;GACrE,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;GACtE;EACF;;;AAIH,MAAa,WAAW,aAA0D;CAChF,QAAQ,YAAY,OAAO,QAAQ,QAAQ,WAAW,EAAE,QAAQ,QAAQ;CACxE,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACjF,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;EACtE;CACF;;AAGD,MAAa,UACX,aACiC;CACjC,QAAQ,YAAY,OAAO,SAAS,QAAQ,QAAQ,QAAQ,WAAW,EAAE,QAAQ,QAAQ;CACzF,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,QAAQ,QAAQ;EAChB,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACjF,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;EACtE;CACF;;AAGD,MAAa,UACX,aAC4C;CAC5C,QAAQ,OAAO,OAAO,OAAO,SAAS,OAAO,eAAe,EAAE,QAAQ,WAAW;CACjF,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EAClF;CACF;;AAGD,MAAa,YAAe,WAA0D;CACpF,QAAQ,OAAO,OAAO,MAAM,OAAO;CACnC,MAAM;EACJ,GAAG,MAAM;EACT,UAAU;EACX;CACF;;;;;;;;;;;;;;;;;AAkBD,MAAa,UACX,WAC+B;CAC/B,MAAM,OAAO,OAAO,QAAQ,OAAO,CAAC,KAAK,CAAC,MAAM,YAAY;EAAE;EAAM,GAAG,MAAM;EAAM,EAAE;CACrF,MAAM,gBAAgB,KAAK,MAAM,UAAU,qBAAqB,MAAM,WAAW,CAAC;AAClF,KAAI,kBAAkB,OACpB,OAAM,IAAI,MAAM,4CAA4C,cAAc,aAAa;CAGzF,MAAM,eAAe,OAAO,IAAI,QAAQ,oBAAoB;AAE5D,QAAO;EACL,QAAQ;EAGR,QAAQ,OAAO,OAAO,aAA8C;EACpE;EACD;;;;;;;;;;;;;;;;;;ACzMH,MAAaA,YAAoD,aAC/D;;;;ACQF,MAAM,kBAAkB,UAAiE;CACvF,MAAM,eAAe,iBAAiB,MAAM;AAC5C,QAAO,OAAO,MAAM,CAClB,cACA,OAAO,eAAe,KACpB,OAAO,SAAS,cAAc,qBAAqB,iBAAiB,CACrE,CACF,CAAC;;AAGJ,MAAM,kBAAkB,OAAO,MAAM,CACnC,OAAO,SACP,OAAO,SAAS,CAAC,QAAQ,QAAQ,CAAC,CAAC,KACjC,OAAO,SACL,OAAO,SACP,qBAAqB,UAAU;CAC7B,SAAS,UAAU,UAAU;CAC7B,SAAS,UAAW,QAAQ,SAAS;CACtC,CAAC,CACH,CACF,CACF,CAAC;AAGF,MAAM,yBAAyB,OAAO,QAAQ,GAAG,CAAC,KAChD,OAAO,SACL,OAAO,WACP,qBAAqB,UAAU;CAAE,cAAc;CAAW,cAAc;CAAI,CAAC,CAC9E,CACF;AAED,MAAM,mBAAmB,UAA2D;AAClF,SAAQ,MAAM,MAAd;EACE,KAAK,SACH,QAAO,eAAe,MAAM;EAC9B,KAAK,UACH,QAAO;EACT,KAAK,SACH,QAAO,OAAO,SAAS,MAAM,OAAO;EACtC,KAAK,SACH,QAAO,OAAO;;;AAIpB,MAAM,eAAe,UACnB,MAAM,WACF,gBAAgB,MAAM,GACtB,OAAO,SAAS,OAAO,MAAM,CAAC,wBAAwB,gBAAgB,MAAM,CAAC,CAAC,CAAC;;;;;;;;;;;;;AAcrF,MAAa,gBACX,aACoC;CACpC,MAAM,SAAS,OAAO,YACpB,SAAS,OAAO,KAAK,UAAU,CAAC,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC,CACjE;AAGD,QAAO,OAAO,OAAO,OAAO;;;AAI9B,MAAa,gBACX,UACA,UACG;CACH,MAAM,WAAW,OAAO,YACtB,SAAS,OAAO,SAAS,UACvB,MAAM,YAAY,SAAY,EAAE,GAAG,CAAC,CAAC,MAAM,MAAM,MAAM,QAAQ,CAAC,CACjE,CACF;CACD,MAAM,iBAAiB,IAAI,IACzB,SAAS,OAAO,SAAS,UAAW,MAAM,WAAW,EAAE,GAAG,CAAC,MAAM,KAAK,CAAE,CACzE;CACD,MAAM,kBAAkB,UAAU,SAAS,MAAM,GAC7C,OAAO,YACL,OAAO,QAAQ,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,UAAU,MAAM,CAAC,eAAe,IAAI,KAAK,CAAC,CAC3F,GACD;CACJ,MAAM,oBAAoB,UAAU,SAAS,gBAAgB,GACzD,OAAO,OAAO,EAAE,EAAE,UAAU,gBAAgB,GAC5C;AAEJ,QAAO,OAAO,oBAAoB,aAAa,SAAS,CAAC,CAAC,mBAAmB,EAC3E,kBAAkB,SACnB,CAAC;;;;;;;;;;;;AAaJ,MAAa,qBACX,UACA,WAC0B;CAC1B,MAAMC,WAAsD,EAAE;AAC9D,MAAK,MAAM,SAAS,SAAS,QAAQ;EACnC,MAAMC,QAAiB,QAAQ,IAAI,QAAQ,MAAM,KAAK;AACtD,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,UAC7E,UAAS,MAAM,cAAc;;AAGjC,QAAO"}
|