@useairfoil/connector-kit 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/METRICS.md +16 -17
- package/README.md +127 -442
- package/dist/annotations-BE74YmAO.d.mts +39 -0
- package/dist/auth/index.d.mts +2 -0
- package/dist/{auth-DvU_CZ7C.js → auth/index.mjs} +6 -8
- package/dist/auth/index.mjs.map +1 -0
- package/dist/connector-app/index.d.mts +2 -0
- package/dist/connector-app/index.mjs +2 -0
- package/dist/connector-app-C-L4QqG6.mjs +123 -0
- package/dist/connector-app-C-L4QqG6.mjs.map +1 -0
- package/dist/{constants-D113Ut1x.js → constants-D-GQzpHZ.mjs} +4 -5
- package/dist/constants-D-GQzpHZ.mjs.map +1 -0
- package/dist/demo/index.d.mts +2 -0
- package/dist/demo/index.mjs +2 -0
- package/dist/demo-B30Vlkk2.mjs +191 -0
- package/dist/demo-B30Vlkk2.mjs.map +1 -0
- package/dist/errors/index.d.mts +2 -0
- package/dist/errors/index.mjs +2 -0
- package/dist/{errors-DHWylqRZ.js → errors-pT0YVJU5.mjs} +2 -3
- package/dist/errors-pT0YVJU5.mjs.map +1 -0
- package/dist/iceberg/index.d.mts +3 -0
- package/dist/iceberg/index.mjs +3 -0
- package/dist/iceberg-D6L8I6vA.mjs +39 -0
- package/dist/iceberg-D6L8I6vA.mjs.map +1 -0
- package/dist/{index-VpHaZXZw.d.ts → index-BUHSGm9Q.d.mts} +11 -13
- package/dist/{index-DXa26Ry5.d.ts → index-BX8G7d41.d.mts} +1 -2
- package/dist/{index-bfjH7Yb0.d.ts → index-BfXpgxk9.d.mts} +9 -10
- package/dist/{index-BKw7jA2a.d.ts → index-CRVAwJ5-.d.mts} +10 -8
- package/dist/index-CUPP2A_m.d.mts +27 -0
- package/dist/{index-DYpEMJf-.d.ts → index-CgvFomO6.d.mts} +6 -7
- package/dist/index-DJhDp4Xc.d.mts +58 -0
- package/dist/index-Dk7urvpF.d.mts +76 -0
- package/dist/{index-DKoRwyFn.d.ts → index-DkVNO6ks.d.mts} +2 -5
- package/dist/{index-C5Ma7V7t.d.ts → index-NnCutIqM.d.mts} +62 -37
- package/dist/{index-CR9HJDdr.d.ts → index-eFuoMZZ3.d.mts} +18 -20
- package/dist/{index-ZSuiZX5H.d.ts → index-ygpml2kC.d.mts} +19 -31
- package/dist/index.d.mts +16 -0
- package/dist/index.mjs +45 -0
- package/dist/index.mjs.map +1 -0
- package/dist/ingestion/index.d.mts +2 -0
- package/dist/ingestion/index.mjs +2 -0
- package/dist/ingestion-q0WwqUSt.mjs +275 -0
- package/dist/ingestion-q0WwqUSt.mjs.map +1 -0
- package/dist/ingestor/index.d.mts +3 -0
- package/dist/ingestor/index.mjs +4 -0
- package/dist/ingestor-D4L-HPkg.mjs +26 -0
- package/dist/ingestor-D4L-HPkg.mjs.map +1 -0
- package/dist/instrumented-Bhp7fyvh.mjs +29 -0
- package/dist/instrumented-Bhp7fyvh.mjs.map +1 -0
- package/dist/manifest/index.d.mts +2 -0
- package/dist/manifest/{index.js → index.mjs} +2 -4
- package/dist/{manifest-P7kBnch9.js → manifest-BGKxmKLy.mjs} +14 -19
- package/dist/manifest-BGKxmKLy.mjs.map +1 -0
- package/dist/{telemetry-j_nxwYfh.js → metrics-BaEX__MN.mjs} +170 -19
- package/dist/metrics-BaEX__MN.mjs.map +1 -0
- package/dist/{metrics-C4V1pm6v.d.ts → metrics-CVXFL7YT.d.mts} +18 -13
- package/dist/metrics.d.mts +2 -0
- package/dist/metrics.mjs +2 -0
- package/dist/{chunk-Bp6m_JJh.js → rolldown-runtime-D7D4PA-g.mjs} +4 -4
- package/dist/runtime-config/index.d.mts +2 -0
- package/dist/runtime-config/index.mjs +3 -0
- package/dist/runtime-config-Oa55gAew.mjs +97 -0
- package/dist/runtime-config-Oa55gAew.mjs.map +1 -0
- package/dist/server-CMnxz3h0.mjs +144 -0
- package/dist/server-CMnxz3h0.mjs.map +1 -0
- package/dist/service-DLkhuxYl.d.mts +27 -0
- package/dist/service-jWxxfEht.mjs +7 -0
- package/dist/service-jWxxfEht.mjs.map +1 -0
- package/dist/state-store/index.d.mts +2 -0
- package/dist/state-store/index.mjs +2 -0
- package/dist/{state-store-BdYulQOY.js → state-store-uFvHY7Ax.mjs} +15 -22
- package/dist/state-store-uFvHY7Ax.mjs.map +1 -0
- package/dist/status/{index.d.ts → index.d.mts} +1 -4
- package/dist/status/{index.js → index.mjs} +2 -6
- package/dist/{status-BTTd7AUR.js → status-CKb9IhQ9.mjs} +14 -16
- package/dist/status-CKb9IhQ9.mjs.map +1 -0
- package/dist/table-V0tgUC5i.mjs +370 -0
- package/dist/table-V0tgUC5i.mjs.map +1 -0
- package/dist/types-xZtSmrGA.d.mts +220 -0
- package/dist/webhook/index.d.mts +3 -0
- package/dist/webhook/index.mjs +12 -0
- package/dist/webhook/index.mjs.map +1 -0
- package/dist/wings-I56hGtNv.mjs +325 -0
- package/dist/wings-I56hGtNv.mjs.map +1 -0
- package/package.json +64 -51
- package/src/catalog/table.ts +201 -0
- package/src/connector-app/create-table.ts +47 -0
- package/src/connector-app/demo.ts +42 -0
- package/src/connector-app/index.ts +2 -0
- package/src/connector-app/start.ts +1 -1
- package/src/core/builder.ts +15 -23
- package/src/core/index.ts +11 -1
- package/src/core/types.ts +114 -39
- package/src/demo/definition.ts +68 -0
- package/src/demo/generate.ts +167 -0
- package/src/demo/index.ts +3 -0
- package/src/demo/run.ts +142 -0
- package/src/iceberg/annotations.ts +62 -0
- package/src/iceberg/error.ts +9 -0
- package/src/iceberg/index.ts +3 -0
- package/src/iceberg/table.ts +511 -0
- package/src/index.ts +3 -1
- package/src/ingestion/engine.ts +240 -60
- package/src/ingestor/arrow.ts +248 -0
- package/src/ingestor/console.ts +13 -0
- package/src/ingestor/index.ts +9 -0
- package/src/ingestor/instrumented.ts +61 -0
- package/src/ingestor/service.ts +30 -0
- package/src/ingestor/wings.ts +143 -0
- package/src/manifest/config.ts +3 -3
- package/src/metrics.ts +66 -20
- package/src/runtime-config/config.ts +6 -8
- package/src/runtime-config/constants.ts +2 -2
- package/src/runtime-config/index.ts +2 -7
- package/src/runtime-config/layer.ts +1 -1
- package/src/runtime-config/table-bindings.ts +48 -136
- package/src/state-store/layer-key-value-store.ts +8 -3
- package/src/state-store/layer-memory.ts +1 -1
- package/src/state-store/layer-sql.ts +4 -4
- package/src/state-store/schema.ts +2 -2
- package/src/state-store/state.ts +14 -2
- package/src/status/route.ts +2 -2
- package/src/telemetry.ts +10 -10
- package/src/webhook/server.ts +39 -40
- package/dist/auth/index.d.ts +0 -2
- package/dist/auth/index.js +0 -3
- package/dist/auth-DvU_CZ7C.js.map +0 -1
- package/dist/connector-app/index.d.ts +0 -14
- package/dist/connector-app/index.js +0 -12
- package/dist/connector-app-BljHd0Pe.js +0 -64
- package/dist/connector-app-BljHd0Pe.js.map +0 -1
- package/dist/constants-D113Ut1x.js.map +0 -1
- package/dist/errors/index.d.ts +0 -2
- package/dist/errors/index.js +0 -3
- package/dist/errors-DHWylqRZ.js.map +0 -1
- package/dist/index-BA8ASHKr.d.ts +0 -31
- package/dist/index.d.ts +0 -14
- package/dist/index.js +0 -60
- package/dist/index.js.map +0 -1
- package/dist/ingestion/index.d.ts +0 -6
- package/dist/ingestion/index.js +0 -11
- package/dist/ingestion-QBGKrFr9.js +0 -200
- package/dist/ingestion-QBGKrFr9.js.map +0 -1
- package/dist/manifest/index.d.ts +0 -2
- package/dist/manifest-P7kBnch9.js.map +0 -1
- package/dist/metrics-inkCAxKq.js +0 -136
- package/dist/metrics-inkCAxKq.js.map +0 -1
- package/dist/metrics.d.ts +0 -4
- package/dist/metrics.js +0 -5
- package/dist/publisher/index.d.ts +0 -6
- package/dist/publisher/index.js +0 -8
- package/dist/publisher-BdWJOfer.js +0 -138
- package/dist/publisher-BdWJOfer.js.map +0 -1
- package/dist/runtime-config/index.d.ts +0 -4
- package/dist/runtime-config/index.js +0 -5
- package/dist/runtime-config-IbyCVpei.js +0 -60
- package/dist/runtime-config-IbyCVpei.js.map +0 -1
- package/dist/server-D-eLX3yN.js +0 -183
- package/dist/server-D-eLX3yN.js.map +0 -1
- package/dist/service-CUjGhe-7.d.ts +0 -32
- package/dist/service-DiP-XETo.js +0 -8
- package/dist/service-DiP-XETo.js.map +0 -1
- package/dist/state-store/index.d.ts +0 -3
- package/dist/state-store/index.js +0 -5
- package/dist/state-store-BdYulQOY.js.map +0 -1
- package/dist/status-BTTd7AUR.js.map +0 -1
- package/dist/table-bindings-Dp1DhTeS.js +0 -106
- package/dist/table-bindings-Dp1DhTeS.js.map +0 -1
- package/dist/telemetry-j_nxwYfh.js.map +0 -1
- package/dist/types-jVD5Ys43.d.ts +0 -174
- package/dist/webhook/index.d.ts +0 -7
- package/dist/webhook/index.js +0 -9
- package/dist/webhook-MhFcLW-o.js +0 -13
- package/dist/webhook-MhFcLW-o.js.map +0 -1
- package/src/publisher/console.ts +0 -18
- package/src/publisher/index.ts +0 -7
- package/src/publisher/instrumented.ts +0 -75
- package/src/publisher/service.ts +0 -37
- package/src/publisher/wings.ts +0 -204
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime-config-Oa55gAew.mjs","names":[],"sources":["../src/runtime-config/config.ts","../src/runtime-config/error.ts","../src/runtime-config/layer.ts","../src/runtime-config/table-bindings.ts","../src/runtime-config/index.ts"],"sourcesContent":["import { Config } from \"effect\";\n\nimport { PlatformRuntimeDefault, PlatformRuntimeKey } from \"./constants\";\n\n/** Shared HTTP port used by hosted and sandbox connector entrypoints. */\nexport const httpPort = Config.Port(PlatformRuntimeKey.httpPort).pipe(\n Config.withDefault(PlatformRuntimeDefault.httpPort),\n);\n\n/** Wings URL, including the protocol. */\nexport const wingsUri = Config.URL(PlatformRuntimeKey.wingsUri);\n\n/** Wings catalog ID. */\nexport const catalog = Config.NonEmptyString(PlatformRuntimeKey.catalog);\n","import { Data } from \"effect\";\n\nexport type RuntimeConfigErrorCode =\n | \"CONFIG_PATH_MISSING\"\n | \"CONFIG_FILE_UNREADABLE\"\n | \"CONFIG_JSON_INVALID\"\n | \"TABLE_BINDINGS_MISSING\"\n | \"TABLE_BINDINGS_INVALID\"\n | \"TABLE_BINDINGS_MISMATCH\";\n\n/** Expected hosted-config failure with a stable code and diagnostic cause. */\nexport class RuntimeConfigError extends Data.TaggedError(\"RuntimeConfigError\")<{\n readonly code: RuntimeConfigErrorCode;\n readonly message: string;\n readonly cause: Error;\n}> {}\n","import { Config, ConfigProvider, Effect, FileSystem, Schema } from \"effect\";\n\nimport { PlatformRuntimeKey } from \"./constants\";\nimport { RuntimeConfigError } from \"./error\";\n\nconst providerFromFile: Effect.Effect<\n ConfigProvider.ConfigProvider,\n RuntimeConfigError,\n FileSystem.FileSystem\n> = Effect.gen(function* () {\n const configPath = yield* Config.NonEmptyString(PlatformRuntimeKey.configPath).pipe(\n Effect.mapError(\n (error) =>\n new RuntimeConfigError({\n code: \"CONFIG_PATH_MISSING\",\n message: `${PlatformRuntimeKey.configPath} is required in hosted mode`,\n cause: error.cause,\n }),\n ),\n );\n const fileSystem = yield* FileSystem.FileSystem;\n const contents = yield* fileSystem.readFileString(configPath).pipe(\n Effect.mapError(\n (cause) =>\n new RuntimeConfigError({\n code: \"CONFIG_FILE_UNREADABLE\",\n message: \"Connector config file could not be read\",\n cause,\n }),\n ),\n );\n const document = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(\n contents,\n ).pipe(\n Effect.mapError(\n (error) =>\n new RuntimeConfigError({\n code: \"CONFIG_JSON_INVALID\",\n message: \"Connector config is not valid JSON\",\n cause: error,\n }),\n ),\n );\n return ConfigProvider.fromUnknown(document, { preserveEmptyStrings: true });\n});\n\n/**\n * Loads the required JSON file and lets environment values override it per key.\n * Connector field validation remains in the connector's Effect Config.\n *\n * @example\n * ```ts\n * program.pipe(Effect.provide(layerHosted()));\n * ```\n */\nexport const layerHosted = () => ConfigProvider.layerAdd(providerFromFile);\n","import type { TableIdentifier } from \"iceberg-js\";\n\nimport { Config, Effect, Schema } from \"effect\";\n\nimport type { TableSource } from \"../core/types\";\n\nimport { PlatformRuntimeKey } from \"./constants\";\nimport { RuntimeConfigError } from \"./error\";\n\n/** An Iceberg table and its storage location for each connector resource. */\nexport type TableBinding = TableIdentifier & { readonly location?: string | undefined };\n\nexport type ResolvedTableBindings<Source extends TableSource> = Readonly<\n Partial<Record<Source[\"resources\"][number][\"name\"], TableBinding>>\n>;\n\nconst TableIdentifierSchema = Schema.Struct({\n namespace: Schema.NonEmptyArray(Schema.NonEmptyString),\n name: Schema.NonEmptyString,\n location: Schema.optional(Schema.NonEmptyString),\n});\n\nconst RawBindingsSchema = Schema.fromJsonString(\n Schema.Record(Schema.String, TableIdentifierSchema),\n);\n\nconst asError = (cause: unknown): Error =>\n cause instanceof Error ? cause : new Error(String(cause));\n\nconst fail = (code: \"TABLE_BINDINGS_INVALID\" | \"TABLE_BINDINGS_MISMATCH\", message: string) =>\n new RuntimeConfigError({ code, message, cause: new Error(message) });\n\n/** Reads hosted table bindings and checks they name resources of the connector. */\nexport const loadTableBindings = <const Source extends TableSource>(\n connector: Source,\n): Effect.Effect<ResolvedTableBindings<Source>, RuntimeConfigError> =>\n Effect.gen(function* () {\n const raw = yield* Config.NonEmptyString(PlatformRuntimeKey.tableBindings).pipe(\n Effect.mapError(\n (cause) =>\n new RuntimeConfigError({\n code: \"TABLE_BINDINGS_MISSING\",\n message: `${PlatformRuntimeKey.tableBindings} is required in hosted mode`,\n cause: asError(cause),\n }),\n ),\n );\n\n const decoded = yield* Schema.decodeUnknownEffect(RawBindingsSchema)(raw, {\n onExcessProperty: \"error\",\n }).pipe(\n Effect.mapError(\n (cause) =>\n new RuntimeConfigError({\n code: \"TABLE_BINDINGS_INVALID\",\n message: \"Hosted table bindings are invalid\",\n cause: asError(cause),\n }),\n ),\n );\n\n const actual = Object.keys(decoded);\n if (actual.length === 0) {\n return yield* Effect.fail(\n fail(\"TABLE_BINDINGS_INVALID\", \"Hosted table bindings name no resources\"),\n );\n }\n\n // Bindings come from the platform, resources from the connector. An unknown name\n // means one of them is out of date, and we could write to the wrong table.\n const expected = new Set(connector.resources.map((resource) => resource.name));\n const unknown = actual.filter((name) => !expected.has(name));\n if (unknown.length > 0) {\n return yield* Effect.fail(\n fail(\n \"TABLE_BINDINGS_MISMATCH\",\n `Hosted table bindings name resources the connector doesn't have (${unknown.length})`,\n ),\n );\n }\n\n return Object.fromEntries(\n Object.entries(decoded).map(([resource, identifier]) => [\n resource,\n {\n namespace: Array.from(identifier.namespace),\n name: identifier.name,\n ...(identifier.location === undefined ? {} : { location: identifier.location }),\n },\n ]),\n ) as ResolvedTableBindings<Source>;\n });\n","export { RuntimeConfigError, type RuntimeConfigErrorCode } from \"./error\";\nexport { catalog, httpPort, wingsUri } from \"./config\";\nexport { PlatformRuntimeDefault, PlatformRuntimeKey } from \"./constants\";\nexport { layerHosted } from \"./layer\";\nexport { type ResolvedTableBindings, type TableBinding, loadTableBindings } from \"./table-bindings\";\n"],"mappings":";;;;;AAKA,MAAa,WAAW,OAAO,KAAK,mBAAmB,QAAQ,CAAC,CAAC,KAC/D,OAAO,YAAY,uBAAuB,QAAQ,CACpD;;AAGA,MAAa,WAAW,OAAO,IAAI,mBAAmB,QAAQ;;AAG9D,MAAa,UAAU,OAAO,eAAe,mBAAmB,OAAO;;;;ACFvE,IAAa,qBAAb,cAAwC,KAAK,YAAY,oBAAoB,CAAC,CAI3E,CAAC;;;ACVJ,MAAM,mBAIF,OAAO,IAAI,aAAa;CAC1B,MAAM,aAAa,OAAO,OAAO,eAAe,mBAAmB,UAAU,CAAC,CAAC,KAC7E,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS,GAAG,mBAAmB,WAAW;EAC1C,OAAO,MAAM;CACf,CAAC,CACL,CACF;CAEA,MAAM,WAAW,QAAO,OADE,WAAW,WAAA,CACF,eAAe,UAAU,CAAC,CAAC,KAC5D,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS;EACT;CACF,CAAC,CACL,CACF;CACA,MAAM,WAAW,OAAO,OAAO,oBAAoB,OAAO,eAAe,OAAO,OAAO,CAAC,CAAC,CACvF,QACF,CAAC,CAAC,KACA,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS;EACT,OAAO;CACT,CAAC,CACL,CACF;CACA,OAAO,eAAe,YAAY,UAAU,EAAE,sBAAsB,KAAK,CAAC;AAC5E,CAAC;;;;;;;;;;AAWD,MAAa,oBAAoB,eAAe,SAAS,gBAAgB;;;ACvCzE,MAAM,wBAAwB,OAAO,OAAO;CAC1C,WAAW,OAAO,cAAc,OAAO,cAAc;CACrD,MAAM,OAAO;CACb,UAAU,OAAO,SAAS,OAAO,cAAc;AACjD,CAAC;AAED,MAAM,oBAAoB,OAAO,eAC/B,OAAO,OAAO,OAAO,QAAQ,qBAAqB,CACpD;AAEA,MAAM,WAAW,UACf,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AAE1D,MAAM,QAAQ,MAA4D,YACxE,IAAI,mBAAmB;CAAE;CAAM;CAAS,OAAO,IAAI,MAAM,OAAO;AAAE,CAAC;;AAGrE,MAAa,qBACX,cAEA,OAAO,IAAI,aAAa;CACtB,MAAM,MAAM,OAAO,OAAO,eAAe,mBAAmB,aAAa,CAAC,CAAC,KACzE,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS,GAAG,mBAAmB,cAAc;EAC7C,OAAO,QAAQ,KAAK;CACtB,CAAC,CACL,CACF;CAEA,MAAM,UAAU,OAAO,OAAO,oBAAoB,iBAAiB,CAAC,CAAC,KAAK,EACxE,kBAAkB,QACpB,CAAC,CAAC,CAAC,KACD,OAAO,UACJ,UACC,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS;EACT,OAAO,QAAQ,KAAK;CACtB,CAAC,CACL,CACF;CAEA,MAAM,SAAS,OAAO,KAAK,OAAO;CAClC,IAAI,OAAO,WAAW,GACpB,OAAO,OAAO,OAAO,KACnB,KAAK,0BAA0B,yCAAyC,CAC1E;CAKF,MAAM,WAAW,IAAI,IAAI,UAAU,UAAU,KAAK,aAAa,SAAS,IAAI,CAAC;CAC7E,MAAM,UAAU,OAAO,QAAQ,SAAS,CAAC,SAAS,IAAI,IAAI,CAAC;CAC3D,IAAI,QAAQ,SAAS,GACnB,OAAO,OAAO,OAAO,KACnB,KACE,2BACA,oEAAoE,QAAQ,OAAO,EACrF,CACF;CAGF,OAAO,OAAO,YACZ,OAAO,QAAQ,OAAO,CAAC,CAAC,KAAK,CAAC,UAAU,gBAAgB,CACtD,UACA;EACE,WAAW,MAAM,KAAK,WAAW,SAAS;EAC1C,MAAM,WAAW;EACjB,GAAI,WAAW,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,WAAW,SAAS;CAC/E,CACF,CAAC,CACH;AACF,CAAC"}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
|
|
2
|
+
import { C as annotateError, c as recordWebhookRequest, p as setWebhookQueueDepth, v as Attr, x as SpanName } from "./metrics-BaEX__MN.mjs";
|
|
3
|
+
import { t as ingestBatch } from "./instrumented-Bhp7fyvh.mjs";
|
|
4
|
+
import { Cause, Context, Effect, Queue, Schema } from "effect";
|
|
5
|
+
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http";
|
|
6
|
+
//#region src/webhook/server.ts
|
|
7
|
+
var WebhookQueue = class extends Context.Service()("@useairfoil/connector-kit/WebhookQueue") {};
|
|
8
|
+
const resourceBatch = (rows) => ({ rows });
|
|
9
|
+
const makeRouteContext = (request, rawBody, payload, batches, resourceNames) => ({
|
|
10
|
+
request,
|
|
11
|
+
rawBody,
|
|
12
|
+
payload,
|
|
13
|
+
to: (resource, payload) => Effect.gen(function* () {
|
|
14
|
+
if (!resourceNames.includes(resource.name)) return;
|
|
15
|
+
if (!resource.webhook) return yield* Effect.fail(new ConnectorError({ message: `Resource ${resource.name} does not define a webhook handler` }));
|
|
16
|
+
const rows = yield* resource.webhook.handler({ payload });
|
|
17
|
+
batches.push({
|
|
18
|
+
resource: resource.name,
|
|
19
|
+
batch: resourceBatch(rows)
|
|
20
|
+
});
|
|
21
|
+
})
|
|
22
|
+
});
|
|
23
|
+
const compactBatches = (batches) => {
|
|
24
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
25
|
+
for (const item of batches) {
|
|
26
|
+
const current = grouped.get(item.resource) ?? [];
|
|
27
|
+
grouped.set(item.resource, [...current, ...item.batch.rows]);
|
|
28
|
+
}
|
|
29
|
+
return Array.from(grouped.entries()).map(([resource, rows]) => ({
|
|
30
|
+
resource,
|
|
31
|
+
batch: resourceBatch(rows)
|
|
32
|
+
}));
|
|
33
|
+
};
|
|
34
|
+
const ingestBatches = (batches, options) => Effect.gen(function* () {
|
|
35
|
+
yield* Effect.forEach(batches, (batch) => ingestBatch({
|
|
36
|
+
connector: options.connectorName,
|
|
37
|
+
resource: batch.resource,
|
|
38
|
+
source: "webhook",
|
|
39
|
+
batch: batch.batch
|
|
40
|
+
}), { concurrency: "unbounded" });
|
|
41
|
+
});
|
|
42
|
+
const badRequest = (message) => HttpServerResponse.jsonUnsafe({
|
|
43
|
+
ok: false,
|
|
44
|
+
error: message
|
|
45
|
+
}, { status: 400 });
|
|
46
|
+
const readRawBody = (request) => request.arrayBuffer.pipe(Effect.map((buffer) => new Uint8Array(buffer)));
|
|
47
|
+
const parseJson = (rawBody) => Effect.try({
|
|
48
|
+
try: () => JSON.parse(new TextDecoder().decode(rawBody)),
|
|
49
|
+
catch: (cause) => new ConnectorError({
|
|
50
|
+
message: "Invalid webhook JSON",
|
|
51
|
+
cause
|
|
52
|
+
})
|
|
53
|
+
});
|
|
54
|
+
const makeHandler = (route, options) => Effect.gen(function* () {
|
|
55
|
+
const request = yield* HttpServerRequest.HttpServerRequest;
|
|
56
|
+
const collected = [];
|
|
57
|
+
const rawBodyResult = yield* readRawBody(request).pipe(Effect.match({
|
|
58
|
+
onFailure: () => ({
|
|
59
|
+
_tag: "Error",
|
|
60
|
+
message: "Failed to read webhook body"
|
|
61
|
+
}),
|
|
62
|
+
onSuccess: (value) => ({
|
|
63
|
+
_tag: "Success",
|
|
64
|
+
value
|
|
65
|
+
})
|
|
66
|
+
}));
|
|
67
|
+
if (rawBodyResult._tag === "Error") {
|
|
68
|
+
yield* recordWebhookRequest({
|
|
69
|
+
connector: options.connectorName,
|
|
70
|
+
path: String(route.path),
|
|
71
|
+
outcome: "read_error"
|
|
72
|
+
});
|
|
73
|
+
return badRequest(rawBodyResult.message);
|
|
74
|
+
}
|
|
75
|
+
const jsonResult = yield* parseJson(rawBodyResult.value).pipe(Effect.match({
|
|
76
|
+
onFailure: (error) => ({
|
|
77
|
+
_tag: "Error",
|
|
78
|
+
message: error.message
|
|
79
|
+
}),
|
|
80
|
+
onSuccess: (value) => ({
|
|
81
|
+
_tag: "Success",
|
|
82
|
+
value
|
|
83
|
+
})
|
|
84
|
+
}));
|
|
85
|
+
if (jsonResult._tag === "Error") {
|
|
86
|
+
yield* recordWebhookRequest({
|
|
87
|
+
connector: options.connectorName,
|
|
88
|
+
path: String(route.path),
|
|
89
|
+
outcome: "invalid_json"
|
|
90
|
+
});
|
|
91
|
+
return badRequest(jsonResult.message);
|
|
92
|
+
}
|
|
93
|
+
const payloadResult = yield* Schema.decodeUnknownEffect(route.schema)(jsonResult.value).pipe(Effect.match({
|
|
94
|
+
onFailure: () => ({
|
|
95
|
+
_tag: "Error",
|
|
96
|
+
message: "Invalid webhook payload"
|
|
97
|
+
}),
|
|
98
|
+
onSuccess: (value) => ({
|
|
99
|
+
_tag: "Success",
|
|
100
|
+
value
|
|
101
|
+
})
|
|
102
|
+
}));
|
|
103
|
+
if (payloadResult._tag === "Error") {
|
|
104
|
+
yield* recordWebhookRequest({
|
|
105
|
+
connector: options.connectorName,
|
|
106
|
+
path: String(route.path),
|
|
107
|
+
outcome: "invalid_payload"
|
|
108
|
+
});
|
|
109
|
+
return badRequest(payloadResult.message);
|
|
110
|
+
}
|
|
111
|
+
const response = yield* Effect.withSpan(route.handler(makeRouteContext(request, rawBodyResult.value, payloadResult.value, collected, options.resourceNames)).pipe(Effect.tapError((error) => annotateError("webhook_handle", error))), SpanName.webhookHandle, { attributes: { [Attr.webhookPath]: route.path } });
|
|
112
|
+
const batches = compactBatches(collected);
|
|
113
|
+
if (route.ackMode === "after-enqueue") {
|
|
114
|
+
const queue = yield* WebhookQueue;
|
|
115
|
+
yield* Queue.offerAll(queue.queue, batches);
|
|
116
|
+
const depth = yield* Queue.size(queue.queue);
|
|
117
|
+
yield* setWebhookQueueDepth(options.connectorName, depth);
|
|
118
|
+
yield* recordWebhookRequest({
|
|
119
|
+
connector: options.connectorName,
|
|
120
|
+
path: String(route.path),
|
|
121
|
+
outcome: response.status >= 400 ? "rejected" : "ok"
|
|
122
|
+
});
|
|
123
|
+
return response;
|
|
124
|
+
}
|
|
125
|
+
yield* Effect.withSpan(ingestBatches(batches, { connectorName: options.connectorName }).pipe(Effect.tapError((error) => annotateError("webhook_ingest", error))), SpanName.webhookHandle, { attributes: { [Attr.webhookPath]: route.path } });
|
|
126
|
+
yield* recordWebhookRequest({
|
|
127
|
+
connector: options.connectorName,
|
|
128
|
+
path: String(route.path),
|
|
129
|
+
outcome: response.status >= 400 ? "rejected" : "ok"
|
|
130
|
+
});
|
|
131
|
+
return response;
|
|
132
|
+
}).pipe(Effect.catchCause((cause) => recordWebhookRequest({
|
|
133
|
+
connector: options.connectorName,
|
|
134
|
+
path: String(route.path),
|
|
135
|
+
outcome: "handler_error"
|
|
136
|
+
}).pipe(Effect.andThen(Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`)), Effect.andThen(Effect.succeed(HttpServerResponse.jsonUnsafe({
|
|
137
|
+
ok: false,
|
|
138
|
+
error: "Webhook handler failed"
|
|
139
|
+
}, { status: 500 }))))));
|
|
140
|
+
const router = (routes, options) => HttpRouter.addAll(routes.map((route) => HttpRouter.route("POST", route.path, makeHandler(route, options))));
|
|
141
|
+
//#endregion
|
|
142
|
+
export { router as n, WebhookQueue as t };
|
|
143
|
+
|
|
144
|
+
//# sourceMappingURL=server-CMnxz3h0.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server-CMnxz3h0.mjs","names":["Metrics.recordWebhookRequest","Metrics.setWebhookQueueDepth"],"sources":["../src/webhook/server.ts"],"sourcesContent":["import { Cause, Context, Effect, Queue, Schema } from \"effect\";\nimport { HttpRouter, HttpServerRequest, HttpServerResponse } from \"effect/http\";\n\nimport type {\n ResourceBatch,\n ResourceDefinition,\n WebhookRoute,\n WebhookRouteContext,\n} from \"../core/types\";\n\nimport { ConnectorError } from \"../errors\";\nimport { ingestBatch } from \"../ingestor/instrumented\";\nimport * as Metrics from \"../metrics\";\nimport { Attr, SpanName, annotateError } from \"../telemetry\";\n\nexport type QueuedWebhookBatch = {\n readonly resource: string;\n readonly batch: ResourceBatch;\n};\n\nexport class WebhookQueue extends Context.Service<\n WebhookQueue,\n {\n readonly queue: Queue.Queue<QueuedWebhookBatch>;\n }\n>()(\"@useairfoil/connector-kit/WebhookQueue\") {}\n\nconst resourceBatch = <Row extends object>(rows: ReadonlyArray<Row>): ResourceBatch<Row> => ({\n rows,\n});\n\nconst makeRouteContext = (\n request: HttpServerRequest.HttpServerRequest,\n rawBody: Uint8Array,\n payload: unknown,\n batches: Array<QueuedWebhookBatch>,\n resourceNames: ReadonlyArray<string>,\n): WebhookRouteContext<ReadonlyArray<ResourceDefinition>, unknown> => ({\n request,\n rawBody,\n payload,\n to: (resource, payload) =>\n Effect.gen(function* () {\n if (!resourceNames.includes(resource.name)) return;\n if (!resource.webhook) {\n return yield* Effect.fail(\n new ConnectorError({\n message: `Resource ${resource.name} does not define a webhook handler`,\n }),\n );\n }\n\n // The route already decoded the payload, so don't decode it again.\n const rows = yield* resource.webhook.handler({ payload });\n batches.push({\n resource: resource.name,\n batch: resourceBatch(rows),\n });\n }),\n});\n\nconst compactBatches = (batches: ReadonlyArray<QueuedWebhookBatch>) => {\n const grouped = new Map<string, object[]>();\n for (const item of batches) {\n const current = grouped.get(item.resource) ?? [];\n grouped.set(item.resource, [...current, ...item.batch.rows]);\n }\n return Array.from(grouped.entries()).map(([resource, rows]) => ({\n resource,\n batch: resourceBatch(rows),\n }));\n};\n\nconst ingestBatches = (\n batches: ReadonlyArray<QueuedWebhookBatch>,\n options: { readonly connectorName: string },\n) =>\n Effect.gen(function* () {\n yield* Effect.forEach(\n batches,\n (batch) =>\n ingestBatch({\n connector: options.connectorName,\n resource: batch.resource,\n source: \"webhook\",\n batch: batch.batch,\n }),\n { concurrency: \"unbounded\" },\n );\n });\n\nconst badRequest = (message: string) =>\n HttpServerResponse.jsonUnsafe({ ok: false, error: message }, { status: 400 });\n\nconst readRawBody = (request: HttpServerRequest.HttpServerRequest) =>\n request.arrayBuffer.pipe(Effect.map((buffer) => new Uint8Array(buffer)));\n\nconst parseJson = (rawBody: Uint8Array) =>\n Effect.try({\n try: () => JSON.parse(new TextDecoder().decode(rawBody)),\n catch: (cause) => new ConnectorError({ message: \"Invalid webhook JSON\", cause }),\n });\n\ntype RouterOptions = {\n readonly connectorName: string;\n /** Events for other resources are skipped. */\n readonly resourceNames: ReadonlyArray<string>;\n};\n\nconst makeHandler = (route: WebhookRoute, options: RouterOptions) =>\n Effect.gen(function* () {\n const request = yield* HttpServerRequest.HttpServerRequest;\n const collected: Array<QueuedWebhookBatch> = [];\n\n const rawBodyResult = yield* readRawBody(request).pipe(\n Effect.match({\n onFailure: () => ({ _tag: \"Error\" as const, message: \"Failed to read webhook body\" }),\n onSuccess: (value) => ({ _tag: \"Success\" as const, value }),\n }),\n );\n\n if (rawBodyResult._tag === \"Error\") {\n yield* Metrics.recordWebhookRequest({\n connector: options.connectorName,\n path: String(route.path),\n outcome: \"read_error\",\n });\n return badRequest(rawBodyResult.message);\n }\n\n const jsonResult = yield* parseJson(rawBodyResult.value).pipe(\n Effect.match({\n onFailure: (error) => ({ _tag: \"Error\" as const, message: error.message }),\n onSuccess: (value) => ({ _tag: \"Success\" as const, value }),\n }),\n );\n\n if (jsonResult._tag === \"Error\") {\n yield* Metrics.recordWebhookRequest({\n connector: options.connectorName,\n path: String(route.path),\n outcome: \"invalid_json\",\n });\n return badRequest(jsonResult.message);\n }\n\n const payloadResult = yield* Schema.decodeUnknownEffect(route.schema)(jsonResult.value).pipe(\n Effect.match({\n onFailure: () => ({ _tag: \"Error\" as const, message: \"Invalid webhook payload\" }),\n onSuccess: (value) => ({ _tag: \"Success\" as const, value }),\n }),\n );\n\n if (payloadResult._tag === \"Error\") {\n yield* Metrics.recordWebhookRequest({\n connector: options.connectorName,\n path: String(route.path),\n outcome: \"invalid_payload\",\n });\n return badRequest(payloadResult.message);\n }\n\n const response = yield* Effect.withSpan(\n route\n .handler(\n makeRouteContext(\n request,\n rawBodyResult.value,\n payloadResult.value,\n collected,\n options.resourceNames,\n ),\n )\n .pipe(Effect.tapError((error) => annotateError(\"webhook_handle\", error))),\n SpanName.webhookHandle,\n { attributes: { [Attr.webhookPath]: route.path } },\n );\n\n const batches = compactBatches(collected);\n\n if (route.ackMode === \"after-enqueue\") {\n const queue = yield* WebhookQueue;\n yield* Queue.offerAll(queue.queue, batches);\n const depth = yield* Queue.size(queue.queue);\n yield* Metrics.setWebhookQueueDepth(options.connectorName, depth);\n yield* Metrics.recordWebhookRequest({\n connector: options.connectorName,\n path: String(route.path),\n outcome: response.status >= 400 ? \"rejected\" : \"ok\",\n });\n\n return response;\n }\n\n yield* Effect.withSpan(\n ingestBatches(batches, { connectorName: options.connectorName }).pipe(\n Effect.tapError((error) => annotateError(\"webhook_ingest\", error)),\n ),\n SpanName.webhookHandle,\n { attributes: { [Attr.webhookPath]: route.path } },\n );\n\n yield* Metrics.recordWebhookRequest({\n connector: options.connectorName,\n path: String(route.path),\n outcome: response.status >= 400 ? \"rejected\" : \"ok\",\n });\n\n return response;\n }).pipe(\n Effect.catchCause((cause) =>\n Metrics.recordWebhookRequest({\n connector: options.connectorName,\n path: String(route.path),\n outcome: \"handler_error\",\n }).pipe(\n Effect.andThen(Effect.logWarning(`Webhook handler error: ${Cause.pretty(cause)}`)),\n Effect.andThen(\n Effect.succeed(\n HttpServerResponse.jsonUnsafe(\n { ok: false, error: \"Webhook handler failed\" },\n { status: 500 },\n ),\n ),\n ),\n ),\n ),\n );\n\nexport const router = (routes: ReadonlyArray<WebhookRoute>, options: RouterOptions) =>\n HttpRouter.addAll(\n routes.map((route) => HttpRouter.route(\"POST\", route.path, makeHandler(route, options))),\n );\n"],"mappings":";;;;;;AAoBA,IAAa,eAAb,cAAkC,QAAQ,QAKxC,CAAC,CAAC,wCAAwC,CAAC,CAAC,CAAC;AAE/C,MAAM,iBAAqC,UAAkD,EAC3F,KACF;AAEA,MAAM,oBACJ,SACA,SACA,SACA,SACA,mBACqE;CACrE;CACA;CACA;CACA,KAAK,UAAU,YACb,OAAO,IAAI,aAAa;EACtB,IAAI,CAAC,cAAc,SAAS,SAAS,IAAI,GAAG;EAC5C,IAAI,CAAC,SAAS,SACZ,OAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EACjB,SAAS,YAAY,SAAS,KAAK,oCACrC,CAAC,CACH;EAIF,MAAM,OAAO,OAAO,SAAS,QAAQ,QAAQ,EAAE,QAAQ,CAAC;EACxD,QAAQ,KAAK;GACX,UAAU,SAAS;GACnB,OAAO,cAAc,IAAI;EAC3B,CAAC;CACH,CAAC;AACL;AAEA,MAAM,kBAAkB,YAA+C;CACrE,MAAM,0BAAU,IAAI,IAAsB;CAC1C,KAAK,MAAM,QAAQ,SAAS;EAC1B,MAAM,UAAU,QAAQ,IAAI,KAAK,QAAQ,KAAK,CAAC;EAC/C,QAAQ,IAAI,KAAK,UAAU,CAAC,GAAG,SAAS,GAAG,KAAK,MAAM,IAAI,CAAC;CAC7D;CACA,OAAO,MAAM,KAAK,QAAQ,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,WAAW;EAC9D;EACA,OAAO,cAAc,IAAI;CAC3B,EAAE;AACJ;AAEA,MAAM,iBACJ,SACA,YAEA,OAAO,IAAI,aAAa;CACtB,OAAO,OAAO,QACZ,UACC,UACC,YAAY;EACV,WAAW,QAAQ;EACnB,UAAU,MAAM;EAChB,QAAQ;EACR,OAAO,MAAM;CACf,CAAC,GACH,EAAE,aAAa,YAAY,CAC7B;AACF,CAAC;AAEH,MAAM,cAAc,YAClB,mBAAmB,WAAW;CAAE,IAAI;CAAO,OAAO;AAAQ,GAAG,EAAE,QAAQ,IAAI,CAAC;AAE9E,MAAM,eAAe,YACnB,QAAQ,YAAY,KAAK,OAAO,KAAK,WAAW,IAAI,WAAW,MAAM,CAAC,CAAC;AAEzE,MAAM,aAAa,YACjB,OAAO,IAAI;CACT,WAAW,KAAK,MAAM,IAAI,YAAY,CAAC,CAAC,OAAO,OAAO,CAAC;CACvD,QAAQ,UAAU,IAAI,eAAe;EAAE,SAAS;EAAwB;CAAM,CAAC;AACjF,CAAC;AAQH,MAAM,eAAe,OAAqB,YACxC,OAAO,IAAI,aAAa;CACtB,MAAM,UAAU,OAAO,kBAAkB;CACzC,MAAM,YAAuC,CAAC;CAE9C,MAAM,gBAAgB,OAAO,YAAY,OAAO,CAAC,CAAC,KAChD,OAAO,MAAM;EACX,kBAAkB;GAAE,MAAM;GAAkB,SAAS;EAA8B;EACnF,YAAY,WAAW;GAAE,MAAM;GAAoB;EAAM;CAC3D,CAAC,CACH;CAEA,IAAI,cAAc,SAAS,SAAS;EAClC,OAAOA,qBAA6B;GAClC,WAAW,QAAQ;GACnB,MAAM,OAAO,MAAM,IAAI;GACvB,SAAS;EACX,CAAC;EACD,OAAO,WAAW,cAAc,OAAO;CACzC;CAEA,MAAM,aAAa,OAAO,UAAU,cAAc,KAAK,CAAC,CAAC,KACvD,OAAO,MAAM;EACX,YAAY,WAAW;GAAE,MAAM;GAAkB,SAAS,MAAM;EAAQ;EACxE,YAAY,WAAW;GAAE,MAAM;GAAoB;EAAM;CAC3D,CAAC,CACH;CAEA,IAAI,WAAW,SAAS,SAAS;EAC/B,OAAOA,qBAA6B;GAClC,WAAW,QAAQ;GACnB,MAAM,OAAO,MAAM,IAAI;GACvB,SAAS;EACX,CAAC;EACD,OAAO,WAAW,WAAW,OAAO;CACtC;CAEA,MAAM,gBAAgB,OAAO,OAAO,oBAAoB,MAAM,MAAM,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,KACtF,OAAO,MAAM;EACX,kBAAkB;GAAE,MAAM;GAAkB,SAAS;EAA0B;EAC/E,YAAY,WAAW;GAAE,MAAM;GAAoB;EAAM;CAC3D,CAAC,CACH;CAEA,IAAI,cAAc,SAAS,SAAS;EAClC,OAAOA,qBAA6B;GAClC,WAAW,QAAQ;GACnB,MAAM,OAAO,MAAM,IAAI;GACvB,SAAS;EACX,CAAC;EACD,OAAO,WAAW,cAAc,OAAO;CACzC;CAEA,MAAM,WAAW,OAAO,OAAO,SAC7B,MACG,QACC,iBACE,SACA,cAAc,OACd,cAAc,OACd,WACA,QAAQ,aACV,CACF,CAAC,CACA,KAAK,OAAO,UAAU,UAAU,cAAc,kBAAkB,KAAK,CAAC,CAAC,GAC1E,SAAS,eACT,EAAE,YAAY,GAAG,KAAK,cAAc,MAAM,KAAK,EAAE,CACnD;CAEA,MAAM,UAAU,eAAe,SAAS;CAExC,IAAI,MAAM,YAAY,iBAAiB;EACrC,MAAM,QAAQ,OAAO;EACrB,OAAO,MAAM,SAAS,MAAM,OAAO,OAAO;EAC1C,MAAM,QAAQ,OAAO,MAAM,KAAK,MAAM,KAAK;EAC3C,OAAOC,qBAA6B,QAAQ,eAAe,KAAK;EAChE,OAAOD,qBAA6B;GAClC,WAAW,QAAQ;GACnB,MAAM,OAAO,MAAM,IAAI;GACvB,SAAS,SAAS,UAAU,MAAM,aAAa;EACjD,CAAC;EAED,OAAO;CACT;CAEA,OAAO,OAAO,SACZ,cAAc,SAAS,EAAE,eAAe,QAAQ,cAAc,CAAC,CAAC,CAAC,KAC/D,OAAO,UAAU,UAAU,cAAc,kBAAkB,KAAK,CAAC,CACnE,GACA,SAAS,eACT,EAAE,YAAY,GAAG,KAAK,cAAc,MAAM,KAAK,EAAE,CACnD;CAEA,OAAOA,qBAA6B;EAClC,WAAW,QAAQ;EACnB,MAAM,OAAO,MAAM,IAAI;EACvB,SAAS,SAAS,UAAU,MAAM,aAAa;CACjD,CAAC;CAED,OAAO;AACT,CAAC,CAAC,CAAC,KACD,OAAO,YAAY,UACjBA,qBAA6B;CAC3B,WAAW,QAAQ;CACnB,MAAM,OAAO,MAAM,IAAI;CACvB,SAAS;AACX,CAAC,CAAC,CAAC,KACD,OAAO,QAAQ,OAAO,WAAW,0BAA0B,MAAM,OAAO,KAAK,GAAG,CAAC,GACjF,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CAAE,IAAI;CAAO,OAAO;AAAyB,GAC7C,EAAE,QAAQ,IAAI,CAChB,CACF,CACF,CACF,CACF,CACF;AAEF,MAAa,UAAU,QAAqC,YAC1D,WAAW,OACT,OAAO,KAAK,UAAU,WAAW,MAAM,QAAQ,MAAM,MAAM,YAAY,OAAO,OAAO,CAAC,CAAC,CACzF"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
|
|
2
|
+
import { u as ResourceBatch } from "./types-xZtSmrGA.mjs";
|
|
3
|
+
import { Context, Effect } from "effect";
|
|
4
|
+
//#region src/ingestor/service.d.ts
|
|
5
|
+
type IngestSource = "backfill" | "changes" | "webhook" | "demo";
|
|
6
|
+
/** A resource batch and the source that produced it. */
|
|
7
|
+
type IngestOptions = {
|
|
8
|
+
readonly resource: string;
|
|
9
|
+
readonly source: IngestSource;
|
|
10
|
+
readonly batch: ResourceBatch;
|
|
11
|
+
};
|
|
12
|
+
/** Rows written, and their size as uncompressed Arrow buffers. */
|
|
13
|
+
type IngestResult = {
|
|
14
|
+
readonly rows: number;
|
|
15
|
+
readonly bytes: number;
|
|
16
|
+
};
|
|
17
|
+
/** Handles batches from connector sources. */
|
|
18
|
+
interface IngestorService {
|
|
19
|
+
readonly ingest: (options: IngestOptions) => Effect.Effect<IngestResult, ConnectorError>;
|
|
20
|
+
/** Resources it has tables for. Only these run. Without it, every resource runs. */
|
|
21
|
+
readonly resourceNames?: ReadonlyArray<string>;
|
|
22
|
+
}
|
|
23
|
+
declare const Ingestor_base: Context.ServiceClass<Ingestor, "@useairfoil/connector-kit/Ingestor", IngestorService>;
|
|
24
|
+
declare class Ingestor extends Ingestor_base {}
|
|
25
|
+
//#endregion
|
|
26
|
+
export { IngestorService as a, Ingestor as i, IngestResult as n, IngestSource as r, IngestOptions as t };
|
|
27
|
+
//# sourceMappingURL=service-DLkhuxYl.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service-jWxxfEht.mjs","names":[],"sources":["../src/ingestor/service.ts"],"sourcesContent":["import { Context, type Effect } from \"effect\";\n\nimport type { ResourceBatch } from \"../core/types\";\nimport type { ConnectorError } from \"../errors\";\n\nexport type IngestSource = \"backfill\" | \"changes\" | \"webhook\" | \"demo\";\n\n/** A resource batch and the source that produced it. */\nexport type IngestOptions = {\n readonly resource: string;\n readonly source: IngestSource;\n readonly batch: ResourceBatch;\n};\n\n/** Rows written, and their size as uncompressed Arrow buffers. */\nexport type IngestResult = {\n readonly rows: number;\n readonly bytes: number;\n};\n\n/** Handles batches from connector sources. */\nexport interface IngestorService {\n readonly ingest: (options: IngestOptions) => Effect.Effect<IngestResult, ConnectorError>;\n /** Resources it has tables for. Only these run. Without it, every resource runs. */\n readonly resourceNames?: ReadonlyArray<string>;\n}\n\nexport class Ingestor extends Context.Service<Ingestor, IngestorService>()(\n \"@useairfoil/connector-kit/Ingestor\",\n) {}\n"],"mappings":";;AA2BA,IAAa,WAAb,cAA8B,QAAQ,QAAmC,CAAC,CACxE,oCACF,CAAC,CAAC,CAAC"}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import { t as
|
|
2
|
-
import { t as ConnectorError } from "./errors-
|
|
3
|
-
import { n as PlatformRuntimeKey, t as PlatformRuntimeDefault } from "./constants-
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { t as ConnectorError } from "./errors-pT0YVJU5.mjs";
|
|
3
|
+
import { n as PlatformRuntimeKey, t as PlatformRuntimeDefault } from "./constants-D-GQzpHZ.mjs";
|
|
4
4
|
import { Config, Context, DateTime, Effect, Layer, Option, Schema } from "effect";
|
|
5
|
-
import { KeyValueStore } from "effect/
|
|
6
|
-
import
|
|
7
|
-
|
|
5
|
+
import { KeyValueStore } from "effect/persistence";
|
|
6
|
+
import "effect/sql";
|
|
8
7
|
//#region src/state-store/service.ts
|
|
9
8
|
/**
|
|
10
9
|
* Effect service used by ingestion and status logic for connector checkpoints.
|
|
@@ -19,7 +18,6 @@ import { SqlClient } from "effect/unstable/sql";
|
|
|
19
18
|
* ```
|
|
20
19
|
*/
|
|
21
20
|
var StateStore = class extends Context.Service()("@useairfoil/connector-kit/StateStore") {};
|
|
22
|
-
|
|
23
21
|
//#endregion
|
|
24
22
|
//#region src/state-store/keys.ts
|
|
25
23
|
const resourceComponentKey = (resource, component) => `resource:${resource}:${component}`;
|
|
@@ -35,7 +33,6 @@ const resourceErrorKey = (resource, source) => resourceComponentKey(resource, `e
|
|
|
35
33
|
* ```
|
|
36
34
|
*/
|
|
37
35
|
const connectorInstanceKeyPrefix = (connectorInstanceId) => `connector-instance:${connectorInstanceId}:`;
|
|
38
|
-
|
|
39
36
|
//#endregion
|
|
40
37
|
//#region src/state-store/schema.ts
|
|
41
38
|
const CursorValueSchema = Schema.Union([Schema.String, Schema.Number]);
|
|
@@ -54,12 +51,12 @@ const ChangesStateSchema = Schema.Struct({
|
|
|
54
51
|
const StateSourceSchema = Schema.Literals(["backfill", "changes"]);
|
|
55
52
|
const StateOperationSchema = Schema.Literals([
|
|
56
53
|
"fetch",
|
|
57
|
-
"
|
|
54
|
+
"ingest",
|
|
58
55
|
"checkpoint"
|
|
59
56
|
]);
|
|
60
57
|
const StateErrorCodeSchema = Schema.Literals([
|
|
61
58
|
"fetch_failed",
|
|
62
|
-
"
|
|
59
|
+
"ingest_failed",
|
|
63
60
|
"checkpoint_failed"
|
|
64
61
|
]);
|
|
65
62
|
/** SDK-owned error context safe to persist and expose through status. */
|
|
@@ -80,7 +77,6 @@ const ResourceStateSchema = Schema.Struct({
|
|
|
80
77
|
changes: Schema.optional(ChangesStateSchema),
|
|
81
78
|
lastError: Schema.optional(ResourceErrorSchema)
|
|
82
79
|
});
|
|
83
|
-
|
|
84
80
|
//#endregion
|
|
85
81
|
//#region src/state-store/layer-key-value-store.ts
|
|
86
82
|
const mapStoreError = (operation, resource) => Effect.mapError((cause) => new ConnectorError({
|
|
@@ -96,11 +92,11 @@ const resourceErrorContext = (source, operation) => {
|
|
|
96
92
|
code: "fetch_failed",
|
|
97
93
|
message: `${sourceName} fetch failed`
|
|
98
94
|
};
|
|
99
|
-
case "
|
|
95
|
+
case "ingest": return {
|
|
100
96
|
source,
|
|
101
97
|
operation,
|
|
102
|
-
code: "
|
|
103
|
-
message: `${sourceName}
|
|
98
|
+
code: "ingest_failed",
|
|
99
|
+
message: `${sourceName} ingestion failed`
|
|
104
100
|
};
|
|
105
101
|
case "checkpoint": return {
|
|
106
102
|
source,
|
|
@@ -157,12 +153,10 @@ const layerKeyValueStore = Layer.effect(StateStore)(Effect.gen(function* () {
|
|
|
157
153
|
clearResourceError
|
|
158
154
|
});
|
|
159
155
|
}));
|
|
160
|
-
|
|
161
156
|
//#endregion
|
|
162
157
|
//#region src/state-store/layer-memory.ts
|
|
163
158
|
/** Process-local state for tests and explicit local sandbox commands. */
|
|
164
159
|
const layerMemory = layerKeyValueStore.pipe(Layer.provide(KeyValueStore.layerMemory));
|
|
165
|
-
|
|
166
160
|
//#endregion
|
|
167
161
|
//#region src/state-store/layer-sql.ts
|
|
168
162
|
const scopedKeyValueStore = (connectorInstanceId) => Layer.effect(KeyValueStore.KeyValueStore)(KeyValueStore.KeyValueStore.pipe(Effect.map(KeyValueStore.prefix(connectorInstanceKeyPrefix(connectorInstanceId)))));
|
|
@@ -181,18 +175,17 @@ const scopedKeyValueStore = (connectorInstanceId) => Layer.effect(KeyValueStore.
|
|
|
181
175
|
* ```
|
|
182
176
|
*/
|
|
183
177
|
const layerSql = () => Layer.unwrap(Config.all({
|
|
184
|
-
connectorInstanceId: Config.
|
|
185
|
-
table: Config.
|
|
178
|
+
connectorInstanceId: Config.NonEmptyString(PlatformRuntimeKey.connectorInstanceId),
|
|
179
|
+
table: Config.NonEmptyString(PlatformRuntimeKey.stateTable).pipe(Config.withDefault(PlatformRuntimeDefault.stateTable))
|
|
186
180
|
}).pipe(Effect.map(({ connectorInstanceId, table }) => layerKeyValueStore.pipe(Layer.provide(scopedKeyValueStore(connectorInstanceId).pipe(Layer.provide(KeyValueStore.layerSql({ table }))))))));
|
|
187
|
-
|
|
188
181
|
//#endregion
|
|
189
182
|
//#region src/state-store/index.ts
|
|
190
|
-
var state_store_exports = /* @__PURE__ */
|
|
183
|
+
var state_store_exports = /* @__PURE__ */ __exportAll({
|
|
191
184
|
StateStore: () => StateStore,
|
|
192
185
|
layerMemory: () => layerMemory,
|
|
193
186
|
layerSql: () => layerSql
|
|
194
187
|
});
|
|
195
|
-
|
|
196
188
|
//#endregion
|
|
197
189
|
export { ResourceStateSchema as a, ResourceErrorSchema as i, layerSql as n, StateStore as o, layerMemory as r, state_store_exports as t };
|
|
198
|
-
|
|
190
|
+
|
|
191
|
+
//# sourceMappingURL=state-store-uFvHY7Ax.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"state-store-uFvHY7Ax.mjs","names":[],"sources":["../src/state-store/service.ts","../src/state-store/keys.ts","../src/state-store/schema.ts","../src/state-store/layer-key-value-store.ts","../src/state-store/layer-memory.ts","../src/state-store/layer-sql.ts","../src/state-store/index.ts"],"sourcesContent":["import { Context, Effect } from \"effect\";\n\nimport type {\n PersistedBackfillState,\n PersistedChangesState,\n PersistedResourceState,\n StateOperation,\n StateSource,\n} from \"./schema\";\n\nimport { ConnectorError } from \"../errors\";\n\n/** Connector-facing durable state operations, independent of the storage backend. */\nexport interface StateStoreService {\n readonly getResourceState: (\n resource: string,\n ) => Effect.Effect<PersistedResourceState | undefined, ConnectorError>;\n readonly setBackfillState: (\n resource: string,\n state: PersistedBackfillState,\n ) => Effect.Effect<void, ConnectorError>;\n readonly setChangesState: (\n resource: string,\n state: PersistedChangesState,\n ) => Effect.Effect<void, ConnectorError>;\n readonly setResourceError: (\n resource: string,\n source: StateSource,\n operation: StateOperation,\n ) => Effect.Effect<void, ConnectorError>;\n readonly clearResourceError: (\n resource: string,\n source: StateSource,\n ) => Effect.Effect<void, ConnectorError>;\n}\n\n/**\n * Effect service used by ingestion and status logic for connector checkpoints.\n * Production provides `layerSql`; sandboxes and tests may provide `layerMemory`.\n *\n * @example\n * ```ts\n * const program = Effect.gen(function* () {\n * const state = yield* StateStore;\n * yield* state.setChangesState(\"orders\", { cursor: \"next-page\" });\n * });\n * ```\n */\nexport class StateStore extends Context.Service<StateStore, StateStoreService>()(\n \"@useairfoil/connector-kit/StateStore\",\n) {}\n","import type { StateSource } from \"./schema\";\n\ntype StateComponent = StateSource | `error:${StateSource}`;\n\nexport const resourceComponentKey = (resource: string, component: StateComponent): string =>\n `resource:${resource}:${component}`;\n\nexport const resourceErrorKey = (resource: string, source: StateSource): string =>\n resourceComponentKey(resource, `error:${source}`);\n\n/**\n * Prefix for all state owned by one connector instance. The platform-owned ID\n * stays visible so operators can inspect and clean up shared-table rows directly.\n *\n * @example\n * ```ts\n * connectorInstanceKeyPrefix(\"team-a/orders\");\n * // \"connector-instance:team-a/orders:\"\n * ```\n */\nexport const connectorInstanceKeyPrefix = (connectorInstanceId: string): string =>\n `connector-instance:${connectorInstanceId}:`;\n","import { Schema } from \"effect\";\n\nconst CursorValueSchema = Schema.Union([Schema.String, Schema.Number]);\n\n/** Durable progress for a finite historical backfill. */\nexport const BackfillStateSchema = Schema.Struct({\n cutoff: CursorValueSchema,\n pageCursor: Schema.optional(CursorValueSchema),\n completed: Schema.Boolean,\n lastSuccessAt: Schema.optional(Schema.String),\n});\n\n/** Durable cursor for an ongoing incremental changes stream. */\nexport const ChangesStateSchema = Schema.Struct({\n cursor: CursorValueSchema,\n lastSuccessAt: Schema.optional(Schema.String),\n});\n\nexport const StateSourceSchema = Schema.Literals([\"backfill\", \"changes\"]);\n\nexport const StateOperationSchema = Schema.Literals([\"fetch\", \"ingest\", \"checkpoint\"]);\n\nexport const StateErrorCodeSchema = Schema.Literals([\n \"fetch_failed\",\n \"ingest_failed\",\n \"checkpoint_failed\",\n]);\n\n/** SDK-owned error context safe to persist and expose through status. */\nexport const ResourceErrorContextSchema = Schema.Struct({\n source: StateSourceSchema,\n operation: StateOperationSchema,\n code: StateErrorCodeSchema,\n message: Schema.String,\n});\n\n/** Stable public error summary persisted for status reporting; never a raw cause. */\nexport const ResourceErrorSchema = Schema.Struct({\n ...ResourceErrorContextSchema.fields,\n at: Schema.String,\n});\n\n/** Complete persisted state assembled from independently stored components. */\nexport const ResourceStateSchema = Schema.Struct({\n backfill: Schema.optional(BackfillStateSchema),\n changes: Schema.optional(ChangesStateSchema),\n lastError: Schema.optional(ResourceErrorSchema),\n});\n\nexport type PersistedBackfillState = Schema.Schema.Type<typeof BackfillStateSchema>;\nexport type PersistedChangesState = Schema.Schema.Type<typeof ChangesStateSchema>;\nexport type StateSource = Schema.Schema.Type<typeof StateSourceSchema>;\nexport type StateOperation = Schema.Schema.Type<typeof StateOperationSchema>;\nexport type StateErrorCode = Schema.Schema.Type<typeof StateErrorCodeSchema>;\nexport type ResourceErrorContext = Schema.Schema.Type<typeof ResourceErrorContextSchema>;\nexport type PersistedResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;\nexport type PersistedResourceState = Schema.Schema.Type<typeof ResourceStateSchema>;\n","import { DateTime, Effect, Layer, Option, Schema } from \"effect\";\nimport { KeyValueStore } from \"effect/persistence\";\n\nimport { ConnectorError } from \"../errors\";\nimport { resourceComponentKey, resourceErrorKey } from \"./keys\";\nimport {\n BackfillStateSchema,\n ChangesStateSchema,\n ResourceErrorSchema,\n type PersistedBackfillState,\n type PersistedChangesState,\n type PersistedResourceError,\n type ResourceErrorContext,\n type StateOperation,\n type StateSource,\n} from \"./schema\";\nimport { StateStore } from \"./service\";\n\nconst mapStoreError = (operation: \"read\" | \"write\" | \"clear\", resource: string) =>\n Effect.mapError(\n (cause: KeyValueStore.KeyValueStoreError | Schema.SchemaError) =>\n new ConnectorError({\n message: `Failed to ${operation} state for resource ${resource}`,\n cause,\n }),\n );\n\nconst resourceErrorContext = (\n source: StateSource,\n operation: StateOperation,\n): ResourceErrorContext => {\n const sourceName = source === \"backfill\" ? \"Backfill\" : \"Changes\";\n switch (operation) {\n case \"fetch\":\n return { source, operation, code: \"fetch_failed\", message: `${sourceName} fetch failed` };\n case \"ingest\":\n return {\n source,\n operation,\n code: \"ingest_failed\",\n message: `${sourceName} ingestion failed`,\n };\n case \"checkpoint\":\n return {\n source,\n operation,\n code: \"checkpoint_failed\",\n message: `${sourceName} checkpoint failed`,\n };\n }\n};\n\nconst latestError = (\n backfill: Option.Option<PersistedResourceError>,\n changes: Option.Option<PersistedResourceError>,\n): Option.Option<PersistedResourceError> => {\n if (Option.isNone(backfill)) return changes;\n if (Option.isNone(changes)) return backfill;\n return backfill.value.at >= changes.value.at ? backfill : changes;\n};\n\n/**\n * Implements StateStore over any Effect KeyValueStore. `toSchemaStore` owns\n * serialization and validates persisted values when they are read and written.\n * The caller owns instance scoping; `layerSql` supplies the production prefix.\n */\nexport const layerKeyValueStore: Layer.Layer<StateStore, never, KeyValueStore.KeyValueStore> =\n Layer.effect(StateStore)(\n Effect.gen(function* () {\n const keyValueStore = yield* KeyValueStore.KeyValueStore;\n const backfillStore = KeyValueStore.toSchemaStore(keyValueStore, BackfillStateSchema);\n const changesStore = KeyValueStore.toSchemaStore(keyValueStore, ChangesStateSchema);\n const errorStore = KeyValueStore.toSchemaStore(keyValueStore, ResourceErrorSchema);\n\n const getResourceState = (resource: string) =>\n // Components use separate keys so a cursor update cannot overwrite the\n // independently managed error or backfill state.\n Effect.all(\n {\n backfill: backfillStore.get(resourceComponentKey(resource, \"backfill\")),\n changes: changesStore.get(resourceComponentKey(resource, \"changes\")),\n backfillError: errorStore.get(resourceErrorKey(resource, \"backfill\")),\n changesError: errorStore.get(resourceErrorKey(resource, \"changes\")),\n },\n { concurrency: \"unbounded\" },\n ).pipe(\n Effect.map(({ backfill, changes, backfillError, changesError }) => {\n const lastError = latestError(backfillError, changesError);\n if (Option.isNone(backfill) && Option.isNone(changes) && Option.isNone(lastError)) {\n return undefined;\n }\n return {\n ...(Option.isSome(backfill) ? { backfill: backfill.value } : {}),\n ...(Option.isSome(changes) ? { changes: changes.value } : {}),\n ...(Option.isSome(lastError) ? { lastError: lastError.value } : {}),\n };\n }),\n mapStoreError(\"read\", resource),\n );\n\n const setBackfillState = (resource: string, state: PersistedBackfillState) =>\n backfillStore\n .set(resourceComponentKey(resource, \"backfill\"), state)\n .pipe(mapStoreError(\"write\", resource));\n\n const setChangesState = (resource: string, state: PersistedChangesState) =>\n changesStore\n .set(resourceComponentKey(resource, \"changes\"), state)\n .pipe(mapStoreError(\"write\", resource));\n\n const setResourceError = (resource: string, source: StateSource, operation: StateOperation) =>\n DateTime.now.pipe(\n Effect.map(DateTime.formatIso),\n Effect.flatMap((at) => {\n const error = resourceErrorContext(source, operation);\n return errorStore.set(resourceErrorKey(resource, source), {\n ...error,\n at,\n });\n }),\n mapStoreError(\"write\", resource),\n );\n\n const clearResourceError = (resource: string, source: StateSource) =>\n errorStore\n .remove(resourceErrorKey(resource, source))\n .pipe(mapStoreError(\"clear\", resource));\n\n return StateStore.of({\n getResourceState,\n setBackfillState,\n setChangesState,\n setResourceError,\n clearResourceError,\n });\n }),\n );\n","import { Layer } from \"effect\";\nimport { KeyValueStore } from \"effect/persistence\";\n\nimport type { StateStore } from \"./service\";\n\nimport { layerKeyValueStore } from \"./layer-key-value-store\";\n\n/** Process-local state for tests and explicit local sandbox commands. */\nexport const layerMemory: Layer.Layer<StateStore> = layerKeyValueStore.pipe(\n Layer.provide(KeyValueStore.layerMemory),\n);\n","import { Config, Effect, Layer } from \"effect\";\nimport { KeyValueStore } from \"effect/persistence\";\nimport { SqlClient } from \"effect/sql\";\n\nimport type { StateStore } from \"./service\";\n\nimport { PlatformRuntimeDefault, PlatformRuntimeKey } from \"../runtime-config/constants\";\nimport { connectorInstanceKeyPrefix } from \"./keys\";\nimport { layerKeyValueStore } from \"./layer-key-value-store\";\n\nconst scopedKeyValueStore = (connectorInstanceId: string) =>\n Layer.effect(KeyValueStore.KeyValueStore)(\n KeyValueStore.KeyValueStore.pipe(\n Effect.map(KeyValueStore.prefix(connectorInstanceKeyPrefix(connectorInstanceId))),\n ),\n );\n\n/**\n * Shared SQL state using the immutable connector instance ID supplied by\n * platform-owned Effect configuration. The table defaults to\n * `_airfoil_connectors_state` and can be overridden with `AIRFOIL_STATE_TABLE`.\n * Effect's SQL KeyValueStore owns the table, while `KeyValueStore.prefix`\n * provides application-level isolation between connector instances in that\n * table. The resulting layer exposes only StateStore, never the unscoped\n * KeyValueStore.\n *\n * @example\n * ```ts\n * const StateStoreLive = layerSql().pipe(Layer.provide(PostgresLive));\n * ```\n */\nexport const layerSql = (): Layer.Layer<StateStore, Config.ConfigError, SqlClient.SqlClient> =>\n Layer.unwrap(\n Config.all({\n connectorInstanceId: Config.NonEmptyString(PlatformRuntimeKey.connectorInstanceId),\n table: Config.NonEmptyString(PlatformRuntimeKey.stateTable).pipe(\n Config.withDefault(PlatformRuntimeDefault.stateTable),\n ),\n }).pipe(\n Effect.map(({ connectorInstanceId, table }) =>\n layerKeyValueStore.pipe(\n Layer.provide(\n scopedKeyValueStore(connectorInstanceId).pipe(\n Layer.provide(KeyValueStore.layerSql({ table })),\n ),\n ),\n ),\n ),\n ),\n );\n","export { StateStore, type StateStoreService } from \"./service\";\nexport { layerMemory } from \"./layer-memory\";\nexport { layerSql } from \"./layer-sql\";\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAgDA,IAAa,aAAb,cAAgC,QAAQ,QAAuC,CAAC,CAC9E,sCACF,CAAC,CAAC,CAAC;;;AC9CH,MAAa,wBAAwB,UAAkB,cACrD,YAAY,SAAS,GAAG;AAE1B,MAAa,oBAAoB,UAAkB,WACjD,qBAAqB,UAAU,SAAS,QAAQ;;;;;;;;;;;AAYlD,MAAa,8BAA8B,wBACzC,sBAAsB,oBAAoB;;;ACnB5C,MAAM,oBAAoB,OAAO,MAAM,CAAC,OAAO,QAAQ,OAAO,MAAM,CAAC;;AAGrE,MAAa,sBAAsB,OAAO,OAAO;CAC/C,QAAQ;CACR,YAAY,OAAO,SAAS,iBAAiB;CAC7C,WAAW,OAAO;CAClB,eAAe,OAAO,SAAS,OAAO,MAAM;AAC9C,CAAC;;AAGD,MAAa,qBAAqB,OAAO,OAAO;CAC9C,QAAQ;CACR,eAAe,OAAO,SAAS,OAAO,MAAM;AAC9C,CAAC;AAED,MAAa,oBAAoB,OAAO,SAAS,CAAC,YAAY,SAAS,CAAC;AAExE,MAAa,uBAAuB,OAAO,SAAS;CAAC;CAAS;CAAU;AAAY,CAAC;AAErF,MAAa,uBAAuB,OAAO,SAAS;CAClD;CACA;CACA;AACF,CAAC;;AAGD,MAAa,6BAA6B,OAAO,OAAO;CACtD,QAAQ;CACR,WAAW;CACX,MAAM;CACN,SAAS,OAAO;AAClB,CAAC;;AAGD,MAAa,sBAAsB,OAAO,OAAO;CAC/C,GAAG,2BAA2B;CAC9B,IAAI,OAAO;AACb,CAAC;;AAGD,MAAa,sBAAsB,OAAO,OAAO;CAC/C,UAAU,OAAO,SAAS,mBAAmB;CAC7C,SAAS,OAAO,SAAS,kBAAkB;CAC3C,WAAW,OAAO,SAAS,mBAAmB;AAChD,CAAC;;;AC7BD,MAAM,iBAAiB,WAAuC,aAC5D,OAAO,UACJ,UACC,IAAI,eAAe;CACjB,SAAS,aAAa,UAAU,sBAAsB;CACtD;AACF,CAAC,CACL;AAEF,MAAM,wBACJ,QACA,cACyB;CACzB,MAAM,aAAa,WAAW,aAAa,aAAa;CACxD,QAAQ,WAAR;EACE,KAAK,SACH,OAAO;GAAE;GAAQ;GAAW,MAAM;GAAgB,SAAS,GAAG,WAAW;EAAe;EAC1F,KAAK,UACH,OAAO;GACL;GACA;GACA,MAAM;GACN,SAAS,GAAG,WAAW;EACzB;EACF,KAAK,cACH,OAAO;GACL;GACA;GACA,MAAM;GACN,SAAS,GAAG,WAAW;EACzB;CACJ;AACF;AAEA,MAAM,eACJ,UACA,YAC0C;CAC1C,IAAI,OAAO,OAAO,QAAQ,GAAG,OAAO;CACpC,IAAI,OAAO,OAAO,OAAO,GAAG,OAAO;CACnC,OAAO,SAAS,MAAM,MAAM,QAAQ,MAAM,KAAK,WAAW;AAC5D;;;;;;AAOA,MAAa,qBACX,MAAM,OAAO,UAAU,CAAC,CACtB,OAAO,IAAI,aAAa;CACtB,MAAM,gBAAgB,OAAO,cAAc;CAC3C,MAAM,gBAAgB,cAAc,cAAc,eAAe,mBAAmB;CACpF,MAAM,eAAe,cAAc,cAAc,eAAe,kBAAkB;CAClF,MAAM,aAAa,cAAc,cAAc,eAAe,mBAAmB;CAEjF,MAAM,oBAAoB,aAGxB,OAAO,IACL;EACE,UAAU,cAAc,IAAI,qBAAqB,UAAU,UAAU,CAAC;EACtE,SAAS,aAAa,IAAI,qBAAqB,UAAU,SAAS,CAAC;EACnE,eAAe,WAAW,IAAI,iBAAiB,UAAU,UAAU,CAAC;EACpE,cAAc,WAAW,IAAI,iBAAiB,UAAU,SAAS,CAAC;CACpE,GACA,EAAE,aAAa,YAAY,CAC7B,CAAC,CAAC,KACA,OAAO,KAAK,EAAE,UAAU,SAAS,eAAe,mBAAmB;EACjE,MAAM,YAAY,YAAY,eAAe,YAAY;EACzD,IAAI,OAAO,OAAO,QAAQ,KAAK,OAAO,OAAO,OAAO,KAAK,OAAO,OAAO,SAAS,GAC9E;EAEF,OAAO;GACL,GAAI,OAAO,OAAO,QAAQ,IAAI,EAAE,UAAU,SAAS,MAAM,IAAI,CAAC;GAC9D,GAAI,OAAO,OAAO,OAAO,IAAI,EAAE,SAAS,QAAQ,MAAM,IAAI,CAAC;GAC3D,GAAI,OAAO,OAAO,SAAS,IAAI,EAAE,WAAW,UAAU,MAAM,IAAI,CAAC;EACnE;CACF,CAAC,GACD,cAAc,QAAQ,QAAQ,CAChC;CAEF,MAAM,oBAAoB,UAAkB,UAC1C,cACG,IAAI,qBAAqB,UAAU,UAAU,GAAG,KAAK,CAAC,CACtD,KAAK,cAAc,SAAS,QAAQ,CAAC;CAE1C,MAAM,mBAAmB,UAAkB,UACzC,aACG,IAAI,qBAAqB,UAAU,SAAS,GAAG,KAAK,CAAC,CACrD,KAAK,cAAc,SAAS,QAAQ,CAAC;CAE1C,MAAM,oBAAoB,UAAkB,QAAqB,cAC/D,SAAS,IAAI,KACX,OAAO,IAAI,SAAS,SAAS,GAC7B,OAAO,SAAS,OAAO;EACrB,MAAM,QAAQ,qBAAqB,QAAQ,SAAS;EACpD,OAAO,WAAW,IAAI,iBAAiB,UAAU,MAAM,GAAG;GACxD,GAAG;GACH;EACF,CAAC;CACH,CAAC,GACD,cAAc,SAAS,QAAQ,CACjC;CAEF,MAAM,sBAAsB,UAAkB,WAC5C,WACG,OAAO,iBAAiB,UAAU,MAAM,CAAC,CAAC,CAC1C,KAAK,cAAc,SAAS,QAAQ,CAAC;CAE1C,OAAO,WAAW,GAAG;EACnB;EACA;EACA;EACA;EACA;CACF,CAAC;AACH,CAAC,CACH;;;;AChIF,MAAa,cAAuC,mBAAmB,KACrE,MAAM,QAAQ,cAAc,WAAW,CACzC;;;ACAA,MAAM,uBAAuB,wBAC3B,MAAM,OAAO,cAAc,aAAa,CAAC,CACvC,cAAc,cAAc,KAC1B,OAAO,IAAI,cAAc,OAAO,2BAA2B,mBAAmB,CAAC,CAAC,CAClF,CACF;;;;;;;;;;;;;;;AAgBF,MAAa,iBACX,MAAM,OACJ,OAAO,IAAI;CACT,qBAAqB,OAAO,eAAe,mBAAmB,mBAAmB;CACjF,OAAO,OAAO,eAAe,mBAAmB,UAAU,CAAC,CAAC,KAC1D,OAAO,YAAY,uBAAuB,UAAU,CACtD;AACF,CAAC,CAAC,CAAC,KACD,OAAO,KAAK,EAAE,qBAAqB,YACjC,mBAAmB,KACjB,MAAM,QACJ,oBAAoB,mBAAmB,CAAC,CAAC,KACvC,MAAM,QAAQ,cAAc,SAAS,EAAE,MAAM,CAAC,CAAC,CACjD,CACF,CACF,CACF,CACF,CACF"}
|
|
@@ -1,5 +1,2 @@
|
|
|
1
|
-
import "../index-
|
|
2
|
-
import "../types-jVD5Ys43.js";
|
|
3
|
-
import "../index-DYpEMJf-.js";
|
|
4
|
-
import { a as ResourceStatusSchema, c as StatusUnavailableResponse, d as SyncStateSchema, f as statusRoute, i as ResourceStatus, l as StatusUnavailableResponseSchema, n as ResourceError, o as StatusResponse, r as ResourceErrorSchema, s as StatusResponseSchema, u as SyncState } from "../index-VpHaZXZw.js";
|
|
1
|
+
import { a as ResourceStatusSchema, c as StatusUnavailableResponse, d as SyncStateSchema, f as statusRoute, i as ResourceStatus, l as StatusUnavailableResponseSchema, n as ResourceError, o as StatusResponse, r as ResourceErrorSchema, s as StatusResponseSchema, u as SyncState } from "../index-BUHSGm9Q.mjs";
|
|
5
2
|
export { ResourceError, ResourceErrorSchema, ResourceStatus, ResourceStatusSchema, StatusResponse, StatusResponseSchema, StatusUnavailableResponse, StatusUnavailableResponseSchema, SyncState, SyncStateSchema, statusRoute };
|
|
@@ -1,6 +1,2 @@
|
|
|
1
|
-
import "../
|
|
2
|
-
|
|
3
|
-
import "../state-store-BdYulQOY.js";
|
|
4
|
-
import { a as StatusUnavailableResponseSchema, i as StatusResponseSchema, n as ResourceErrorSchema, o as SyncStateSchema, r as ResourceStatusSchema, s as statusRoute } from "../status-BTTd7AUR.js";
|
|
5
|
-
|
|
6
|
-
export { ResourceErrorSchema, ResourceStatusSchema, StatusResponseSchema, StatusUnavailableResponseSchema, SyncStateSchema, statusRoute };
|
|
1
|
+
import { a as StatusUnavailableResponseSchema, i as StatusResponseSchema, n as ResourceErrorSchema, o as SyncStateSchema, r as ResourceStatusSchema, s as statusRoute } from "../status-CKb9IhQ9.mjs";
|
|
2
|
+
export { ResourceErrorSchema, ResourceStatusSchema, StatusResponseSchema, StatusUnavailableResponseSchema, SyncStateSchema, statusRoute };
|
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
import { t as
|
|
2
|
-
import { a as ResourceStateSchema, i as ResourceErrorSchema, o as StateStore } from "./state-store-
|
|
1
|
+
import { t as __exportAll } from "./rolldown-runtime-D7D4PA-g.mjs";
|
|
2
|
+
import { a as ResourceStateSchema, i as ResourceErrorSchema$1, o as StateStore } from "./state-store-uFvHY7Ax.mjs";
|
|
3
3
|
import { Cause, Effect, Schema } from "effect";
|
|
4
|
-
import { HttpRouter, HttpServerResponse } from "effect/
|
|
5
|
-
|
|
4
|
+
import { HttpRouter, HttpServerResponse } from "effect/http";
|
|
6
5
|
//#region src/state-store/state.ts
|
|
7
6
|
/** Normalizes a cursor value to its persisted/wire representation (never a Date). */
|
|
8
7
|
const normalizeCursor = (value) => value instanceof Date ? value.toISOString() : value;
|
|
8
|
+
/** Whether a resource gets changes, from its own `changes` or the connector's feed. */
|
|
9
|
+
const hasChanges = (connector, resource) => resource.changes !== void 0 || (connector.changes?.resources.some((member) => member.name === resource.name) ?? false);
|
|
9
10
|
/** Derives connector-facing sync state from durable state and resource capabilities. */
|
|
10
|
-
const deriveSyncState = (resource, state) => {
|
|
11
|
+
const deriveSyncState = (connector, resource, state) => {
|
|
11
12
|
if (state?.lastError) return "error";
|
|
12
|
-
if (!resource.backfill && !resource
|
|
13
|
+
if (!resource.backfill && !hasChanges(connector, resource)) return "live";
|
|
13
14
|
if (!state) return "pending";
|
|
14
15
|
if (resource.backfill && (!state.backfill || !state.backfill.completed)) return "backfilling";
|
|
15
16
|
return "live";
|
|
16
17
|
};
|
|
17
|
-
|
|
18
18
|
//#endregion
|
|
19
19
|
//#region src/status/route.ts
|
|
20
20
|
const statusRoute = (connector, path = "/status") => HttpRouter.add("GET", path, Effect.gen(function* () {
|
|
21
21
|
const store = yield* StateStore;
|
|
22
22
|
const resources = yield* Effect.forEach(connector.resources, (resource) => store.getResourceState(resource.name).pipe(Effect.map((state) => ({
|
|
23
23
|
name: resource.name,
|
|
24
|
-
state: deriveSyncState(resource, state),
|
|
24
|
+
state: deriveSyncState(connector, resource, state),
|
|
25
25
|
...state?.backfill ? { backfill: {
|
|
26
26
|
completed: state.backfill.completed,
|
|
27
27
|
cutoff: normalizeCursor(state.backfill.cutoff),
|
|
@@ -42,7 +42,6 @@ const statusRoute = (connector, path = "/status") => HttpRouter.add("GET", path,
|
|
|
42
42
|
ok: false,
|
|
43
43
|
error: "Failed to compute status"
|
|
44
44
|
}, { status: 500 })))))));
|
|
45
|
-
|
|
46
45
|
//#endregion
|
|
47
46
|
//#region src/status/types.ts
|
|
48
47
|
const SyncStateSchema = Schema.Literals([
|
|
@@ -52,7 +51,7 @@ const SyncStateSchema = Schema.Literals([
|
|
|
52
51
|
"error"
|
|
53
52
|
]);
|
|
54
53
|
/** Safe source failure returned by the status endpoint. */
|
|
55
|
-
const ResourceErrorSchema
|
|
54
|
+
const ResourceErrorSchema = ResourceErrorSchema$1;
|
|
56
55
|
const ResourceStatusSchema = Schema.Struct({
|
|
57
56
|
name: Schema.String,
|
|
58
57
|
state: SyncStateSchema,
|
|
@@ -67,18 +66,17 @@ const StatusUnavailableResponseSchema = Schema.Struct({
|
|
|
67
66
|
ok: Schema.Literal(false),
|
|
68
67
|
error: Schema.String
|
|
69
68
|
});
|
|
70
|
-
|
|
71
69
|
//#endregion
|
|
72
70
|
//#region src/status/index.ts
|
|
73
|
-
var status_exports = /* @__PURE__ */
|
|
74
|
-
ResourceErrorSchema: () => ResourceErrorSchema
|
|
71
|
+
var status_exports = /* @__PURE__ */ __exportAll({
|
|
72
|
+
ResourceErrorSchema: () => ResourceErrorSchema,
|
|
75
73
|
ResourceStatusSchema: () => ResourceStatusSchema,
|
|
76
74
|
StatusResponseSchema: () => StatusResponseSchema,
|
|
77
75
|
StatusUnavailableResponseSchema: () => StatusUnavailableResponseSchema,
|
|
78
76
|
SyncStateSchema: () => SyncStateSchema,
|
|
79
77
|
statusRoute: () => statusRoute
|
|
80
78
|
});
|
|
81
|
-
|
|
82
79
|
//#endregion
|
|
83
|
-
export { StatusUnavailableResponseSchema as a, deriveSyncState as c, StatusResponseSchema as i,
|
|
84
|
-
|
|
80
|
+
export { StatusUnavailableResponseSchema as a, deriveSyncState as c, StatusResponseSchema as i, hasChanges as l, ResourceErrorSchema as n, SyncStateSchema as o, ResourceStatusSchema as r, statusRoute as s, status_exports as t, normalizeCursor as u };
|
|
81
|
+
|
|
82
|
+
//# sourceMappingURL=status-CKb9IhQ9.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"status-CKb9IhQ9.mjs","names":["StateResourceErrorSchema"],"sources":["../src/state-store/state.ts","../src/status/route.ts","../src/status/types.ts","../src/status/index.ts"],"sourcesContent":["import type {\n ConnectorDefinition,\n Cursor,\n ResourceDefinition,\n ResourceState,\n SyncState,\n} from \"../core/types\";\n\n/** Normalizes a cursor value to its persisted/wire representation (never a Date). */\nexport const normalizeCursor = (value: Cursor.Value): string | number =>\n value instanceof Date ? value.toISOString() : value;\n\n/** Whether a resource gets changes, from its own `changes` or the connector's feed. */\nexport const hasChanges = (connector: ConnectorDefinition, resource: ResourceDefinition) =>\n resource.changes !== undefined ||\n (connector.changes?.resources.some((member) => member.name === resource.name) ?? false);\n\n/** Derives connector-facing sync state from durable state and resource capabilities. */\nexport const deriveSyncState = (\n connector: ConnectorDefinition,\n resource: ResourceDefinition,\n state: ResourceState | undefined,\n): SyncState => {\n if (state?.lastError) return \"error\";\n if (!resource.backfill && !hasChanges(connector, resource)) return \"live\";\n if (!state) return \"pending\";\n if (resource.backfill && (!state.backfill || !state.backfill.completed)) return \"backfilling\";\n return \"live\";\n};\n","import { Cause, Effect } from \"effect\";\nimport { HttpRouter, HttpServerResponse } from \"effect/http\";\n\nimport type { ConnectorDefinition } from \"../core/types\";\nimport type { ResourceStatus, StatusUnavailableResponse } from \"./types\";\n\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, normalizeCursor } from \"../state-store/state\";\n\nexport const statusRoute = (\n connector: ConnectorDefinition,\n path: HttpRouter.PathInput = \"/status\",\n) =>\n HttpRouter.add(\n \"GET\",\n path,\n Effect.gen(function* () {\n const store = yield* StateStore;\n\n const resources = yield* Effect.forEach(\n connector.resources,\n (resource) =>\n store.getResourceState(resource.name).pipe(\n Effect.map(\n (state): ResourceStatus => ({\n name: resource.name,\n state: deriveSyncState(connector, resource, state),\n ...(state?.backfill\n ? {\n backfill: {\n completed: state.backfill.completed,\n cutoff: normalizeCursor(state.backfill.cutoff),\n ...(state.backfill.pageCursor !== undefined\n ? { pageCursor: normalizeCursor(state.backfill.pageCursor) }\n : {}),\n ...(state.backfill.lastSuccessAt !== undefined\n ? { lastSuccessAt: state.backfill.lastSuccessAt }\n : {}),\n },\n }\n : {}),\n ...(state?.changes\n ? {\n changes: {\n cursor: normalizeCursor(state.changes.cursor),\n ...(state.changes.lastSuccessAt !== undefined\n ? { lastSuccessAt: state.changes.lastSuccessAt }\n : {}),\n },\n }\n : {}),\n ...(state?.lastError ? { lastError: state.lastError } : {}),\n }),\n ),\n ),\n { concurrency: \"unbounded\" },\n );\n\n return HttpServerResponse.jsonUnsafe({ connector: connector.name, resources });\n }).pipe(\n Effect.catchCause((cause) =>\n Effect.logWarning(`Status route failed: ${Cause.pretty(cause)}`).pipe(\n Effect.andThen(\n Effect.succeed(\n HttpServerResponse.jsonUnsafe(\n {\n ok: false,\n error: \"Failed to compute status\",\n } satisfies StatusUnavailableResponse,\n { status: 500 },\n ),\n ),\n ),\n ),\n ),\n ),\n );\n","import { Schema } from \"effect\";\n\nimport {\n ResourceErrorSchema as StateResourceErrorSchema,\n ResourceStateSchema,\n} from \"../state-store/schema\";\n\nexport const SyncStateSchema = Schema.Literals([\"pending\", \"backfilling\", \"live\", \"error\"]);\nexport type SyncState = Schema.Schema.Type<typeof SyncStateSchema>;\n\n/** Safe source failure returned by the status endpoint. */\nexport const ResourceErrorSchema = StateResourceErrorSchema;\nexport type ResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;\n\nexport const ResourceStatusSchema = Schema.Struct({\n name: Schema.String,\n state: SyncStateSchema,\n ...ResourceStateSchema.fields,\n});\nexport type ResourceStatus = Schema.Schema.Type<typeof ResourceStatusSchema>;\n\nexport const StatusResponseSchema = Schema.Struct({\n connector: Schema.String,\n resources: Schema.Array(ResourceStatusSchema),\n});\nexport type StatusResponse = Schema.Schema.Type<typeof StatusResponseSchema>;\n\n/** Returned when durable resource status cannot be read. */\nexport const StatusUnavailableResponseSchema = Schema.Struct({\n ok: Schema.Literal(false),\n error: Schema.String,\n});\nexport type StatusUnavailableResponse = Schema.Schema.Type<typeof StatusUnavailableResponseSchema>;\n","export * from \"./route\";\nexport * from \"./types\";\n"],"mappings":";;;;;;AASA,MAAa,mBAAmB,UAC9B,iBAAiB,OAAO,MAAM,YAAY,IAAI;;AAGhD,MAAa,cAAc,WAAgC,aACzD,SAAS,YAAY,KAAA,MACpB,UAAU,SAAS,UAAU,MAAM,WAAW,OAAO,SAAS,SAAS,IAAI,KAAK;;AAGnF,MAAa,mBACX,WACA,UACA,UACc;CACd,IAAI,OAAO,WAAW,OAAO;CAC7B,IAAI,CAAC,SAAS,YAAY,CAAC,WAAW,WAAW,QAAQ,GAAG,OAAO;CACnE,IAAI,CAAC,OAAO,OAAO;CACnB,IAAI,SAAS,aAAa,CAAC,MAAM,YAAY,CAAC,MAAM,SAAS,YAAY,OAAO;CAChF,OAAO;AACT;;;ACnBA,MAAa,eACX,WACA,OAA6B,cAE7B,WAAW,IACT,OACA,MACA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CAErB,MAAM,YAAY,OAAO,OAAO,QAC9B,UAAU,YACT,aACC,MAAM,iBAAiB,SAAS,IAAI,CAAC,CAAC,KACpC,OAAO,KACJ,WAA2B;EAC1B,MAAM,SAAS;EACf,OAAO,gBAAgB,WAAW,UAAU,KAAK;EACjD,GAAI,OAAO,WACP,EACE,UAAU;GACR,WAAW,MAAM,SAAS;GAC1B,QAAQ,gBAAgB,MAAM,SAAS,MAAM;GAC7C,GAAI,MAAM,SAAS,eAAe,KAAA,IAC9B,EAAE,YAAY,gBAAgB,MAAM,SAAS,UAAU,EAAE,IACzD,CAAC;GACL,GAAI,MAAM,SAAS,kBAAkB,KAAA,IACjC,EAAE,eAAe,MAAM,SAAS,cAAc,IAC9C,CAAC;EACP,EACF,IACA,CAAC;EACL,GAAI,OAAO,UACP,EACE,SAAS;GACP,QAAQ,gBAAgB,MAAM,QAAQ,MAAM;GAC5C,GAAI,MAAM,QAAQ,kBAAkB,KAAA,IAChC,EAAE,eAAe,MAAM,QAAQ,cAAc,IAC7C,CAAC;EACP,EACF,IACA,CAAC;EACL,GAAI,OAAO,YAAY,EAAE,WAAW,MAAM,UAAU,IAAI,CAAC;CAC3D,EACF,CACF,GACF,EAAE,aAAa,YAAY,CAC7B;CAEA,OAAO,mBAAmB,WAAW;EAAE,WAAW,UAAU;EAAM;CAAU,CAAC;AAC/E,CAAC,CAAC,CAAC,KACD,OAAO,YAAY,UACjB,OAAO,WAAW,wBAAwB,MAAM,OAAO,KAAK,GAAG,CAAC,CAAC,KAC/D,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CACE,IAAI;CACJ,OAAO;AACT,GACA,EAAE,QAAQ,IAAI,CAChB,CACF,CACF,CACF,CACF,CACF,CACF;;;ACrEF,MAAa,kBAAkB,OAAO,SAAS;CAAC;CAAW;CAAe;CAAQ;AAAO,CAAC;;AAI1F,MAAa,sBAAsBA;AAGnC,MAAa,uBAAuB,OAAO,OAAO;CAChD,MAAM,OAAO;CACb,OAAO;CACP,GAAG,oBAAoB;AACzB,CAAC;AAGD,MAAa,uBAAuB,OAAO,OAAO;CAChD,WAAW,OAAO;CAClB,WAAW,OAAO,MAAM,oBAAoB;AAC9C,CAAC;;AAID,MAAa,kCAAkC,OAAO,OAAO;CAC3D,IAAI,OAAO,QAAQ,KAAK;CACxB,OAAO,OAAO;AAChB,CAAC"}
|