@useairfoil/connector-kit 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/METRICS.md +78 -0
- package/README.md +450 -88
- package/dist/auth/index.d.ts +2 -0
- package/dist/auth/index.js +3 -0
- package/dist/auth-DvU_CZ7C.js +42 -0
- package/dist/auth-DvU_CZ7C.js.map +1 -0
- package/dist/chunk-Bp6m_JJh.js +13 -0
- package/dist/connector-app/index.d.ts +14 -0
- package/dist/connector-app/index.js +11 -0
- package/dist/connector-app-d7Z-SZLQ.js +64 -0
- package/dist/connector-app-d7Z-SZLQ.js.map +1 -0
- package/dist/constants-BRhIX_rz.js +27 -0
- package/dist/constants-BRhIX_rz.js.map +1 -0
- package/dist/errors/index.d.ts +2 -0
- package/dist/errors/index.js +3 -0
- package/dist/errors-DHWylqRZ.js +8 -0
- package/dist/errors-DHWylqRZ.js.map +1 -0
- package/dist/index-Byr0Lf0C.d.ts +25 -0
- package/dist/index-CBT-JX-T.d.ts +173 -0
- package/dist/index-CTHVudvQ.d.ts +41 -0
- package/dist/index-CTLYQ74P.d.ts +80 -0
- package/dist/index-Coh_SkBs.d.ts +95 -0
- package/dist/index-D54ySAKn.d.ts +14 -0
- package/dist/index-DtJX_tic.d.ts +153 -0
- package/dist/index-J42AZaXU.d.ts +38 -0
- package/dist/index-fIIBVAai.d.ts +25 -0
- package/dist/index-o1-Z3n7V.d.ts +55 -0
- package/dist/index.d.ts +14 -165
- package/dist/index.js +50 -228
- package/dist/index.js.map +1 -1
- package/dist/ingestion/index.d.ts +6 -0
- package/dist/ingestion/index.js +10 -0
- package/dist/ingestion-BQo4cbGw.js +200 -0
- package/dist/ingestion-BQo4cbGw.js.map +1 -0
- package/dist/manifest/index.d.ts +2 -0
- package/dist/manifest/index.js +4 -0
- package/dist/manifest-ByE6YPG0.js +221 -0
- package/dist/manifest-ByE6YPG0.js.map +1 -0
- package/dist/metrics-CVAk1ALR.d.ts +53 -0
- package/dist/metrics-CWPy2Aqc.js +96 -0
- package/dist/metrics-CWPy2Aqc.js.map +1 -0
- package/dist/metrics.d.ts +4 -0
- package/dist/metrics.js +3 -0
- package/dist/publisher/index.d.ts +5 -0
- package/dist/publisher/index.js +6 -0
- package/dist/publisher-BYT3RLVO.js +131 -0
- package/dist/publisher-BYT3RLVO.js.map +1 -0
- package/dist/runtime-config/index.d.ts +2 -0
- package/dist/runtime-config/index.js +4 -0
- package/dist/runtime-config-BpB-d_bF.js +51 -0
- package/dist/runtime-config-BpB-d_bF.js.map +1 -0
- package/dist/server-CXTQ7odl.js +183 -0
- package/dist/server-CXTQ7odl.js.map +1 -0
- package/dist/service-DuFiQwid.js +191 -0
- package/dist/service-DuFiQwid.js.map +1 -0
- package/dist/service-IE5qaf9x.d.ts +32 -0
- package/dist/state-store/index.d.ts +3 -0
- package/dist/state-store/index.js +5 -0
- package/dist/state-store-yl6LiTQ0.js +198 -0
- package/dist/state-store-yl6LiTQ0.js.map +1 -0
- package/dist/status/index.d.ts +5 -0
- package/dist/status/index.js +6 -0
- package/dist/status-B-oZBc33.js +84 -0
- package/dist/status-B-oZBc33.js.map +1 -0
- package/dist/types-B35J5YPu.d.ts +174 -0
- package/dist/webhook/index.d.ts +6 -0
- package/dist/webhook/index.js +8 -0
- package/dist/webhook-DI-Afxkg.js +13 -0
- package/dist/webhook-DI-Afxkg.js.map +1 -0
- package/package.json +66 -8
- package/src/auth/index.ts +1 -0
- package/src/auth/token-cache.ts +54 -0
- package/src/connector-app/check.ts +76 -0
- package/src/connector-app/index.ts +2 -0
- package/src/connector-app/start.ts +42 -0
- package/src/core/builder.ts +81 -9
- package/src/core/index.ts +22 -0
- package/src/core/types.ts +204 -76
- package/src/errors/index.ts +1 -0
- package/src/index.ts +13 -28
- package/src/ingestion/engine.ts +409 -174
- package/src/ingestion/index.ts +1 -0
- package/src/manifest/config.ts +232 -0
- package/src/manifest/index.ts +14 -0
- package/src/manifest/manifest.ts +31 -0
- package/src/manifest/number-schema.ts +17 -0
- package/src/manifest/schema.ts +160 -0
- package/src/metrics.ts +128 -0
- package/src/publisher/console.ts +18 -0
- package/src/publisher/index.ts +7 -0
- package/src/publisher/instrumented.ts +68 -0
- package/src/publisher/service.ts +30 -13
- package/src/publisher/wings.ts +159 -72
- package/src/runtime/context.ts +15 -0
- package/src/runtime-config/constants.ts +27 -0
- package/src/runtime-config/error.ts +13 -0
- package/src/runtime-config/index.ts +3 -0
- package/src/runtime-config/layer.ts +56 -0
- package/src/state-store/index.ts +3 -0
- package/src/state-store/keys.ts +22 -0
- package/src/state-store/layer-key-value-store.ts +132 -0
- package/src/state-store/layer-memory.ts +11 -0
- package/src/state-store/layer-sql.ts +50 -0
- package/src/state-store/schema.ts +57 -0
- package/src/state-store/service.ts +51 -0
- package/src/state-store/state.ts +17 -0
- package/src/status/index.ts +2 -0
- package/src/status/route.ts +77 -0
- package/src/status/types.ts +33 -0
- package/src/telemetry.ts +288 -0
- package/src/webhook/index.ts +8 -0
- package/src/webhook/server.ts +209 -39
- package/src/webhook/types.ts +6 -17
- package/src/ingestion/state-store.ts +0 -36
- package/src/streams/pull-stream.ts +0 -54
- package/src/streams/webhook-queue.ts +0 -14
- /package/src/{core/errors.ts → errors/connector-error.ts} +0 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { t as ConnectorError } from "./errors-DHWylqRZ.js";
|
|
3
|
+
import { n as PlatformRuntimeKey, t as PlatformRuntimeDefault } from "./constants-BRhIX_rz.js";
|
|
4
|
+
import { Config, Context, DateTime, Effect, Layer, Option, Schema } from "effect";
|
|
5
|
+
import { KeyValueStore } from "effect/unstable/persistence";
|
|
6
|
+
import { SqlClient } from "effect/unstable/sql";
|
|
7
|
+
|
|
8
|
+
//#region src/state-store/service.ts
|
|
9
|
+
/**
|
|
10
|
+
* Effect service used by ingestion and status logic for connector checkpoints.
|
|
11
|
+
* Production provides `layerSql`; sandboxes and tests may provide `layerMemory`.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```ts
|
|
15
|
+
* const program = Effect.gen(function* () {
|
|
16
|
+
* const state = yield* StateStore;
|
|
17
|
+
* yield* state.setChangesState("orders", { cursor: "next-page" });
|
|
18
|
+
* });
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
var StateStore = class extends Context.Service()("@useairfoil/connector-kit/StateStore") {};
|
|
22
|
+
|
|
23
|
+
//#endregion
|
|
24
|
+
//#region src/state-store/keys.ts
|
|
25
|
+
const resourceComponentKey = (resource, component) => `resource:${resource}:${component}`;
|
|
26
|
+
const resourceErrorKey = (resource, source) => resourceComponentKey(resource, `error:${source}`);
|
|
27
|
+
/**
|
|
28
|
+
* Prefix for all state owned by one connector instance. The platform-owned ID
|
|
29
|
+
* stays visible so operators can inspect and clean up shared-table rows directly.
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```ts
|
|
33
|
+
* connectorInstanceKeyPrefix("team-a/orders");
|
|
34
|
+
* // "connector-instance:team-a/orders:"
|
|
35
|
+
* ```
|
|
36
|
+
*/
|
|
37
|
+
const connectorInstanceKeyPrefix = (connectorInstanceId) => `connector-instance:${connectorInstanceId}:`;
|
|
38
|
+
|
|
39
|
+
//#endregion
|
|
40
|
+
//#region src/state-store/schema.ts
|
|
41
|
+
const CursorValueSchema = Schema.Union([Schema.String, Schema.Number]);
|
|
42
|
+
/** Durable progress for a finite historical backfill. */
|
|
43
|
+
const BackfillStateSchema = Schema.Struct({
|
|
44
|
+
cutoff: CursorValueSchema,
|
|
45
|
+
pageCursor: Schema.optional(CursorValueSchema),
|
|
46
|
+
completed: Schema.Boolean,
|
|
47
|
+
lastSuccessAt: Schema.optional(Schema.String)
|
|
48
|
+
});
|
|
49
|
+
/** Durable cursor for an ongoing incremental changes stream. */
|
|
50
|
+
const ChangesStateSchema = Schema.Struct({
|
|
51
|
+
cursor: CursorValueSchema,
|
|
52
|
+
lastSuccessAt: Schema.optional(Schema.String)
|
|
53
|
+
});
|
|
54
|
+
const StateSourceSchema = Schema.Literals(["backfill", "changes"]);
|
|
55
|
+
const StateOperationSchema = Schema.Literals([
|
|
56
|
+
"fetch",
|
|
57
|
+
"publish",
|
|
58
|
+
"checkpoint"
|
|
59
|
+
]);
|
|
60
|
+
const StateErrorCodeSchema = Schema.Literals([
|
|
61
|
+
"fetch_failed",
|
|
62
|
+
"publish_failed",
|
|
63
|
+
"checkpoint_failed"
|
|
64
|
+
]);
|
|
65
|
+
/** SDK-owned error context safe to persist and expose through status. */
|
|
66
|
+
const ResourceErrorContextSchema = Schema.Struct({
|
|
67
|
+
source: StateSourceSchema,
|
|
68
|
+
operation: StateOperationSchema,
|
|
69
|
+
code: StateErrorCodeSchema,
|
|
70
|
+
message: Schema.String
|
|
71
|
+
});
|
|
72
|
+
/** Stable public error summary persisted for status reporting; never a raw cause. */
|
|
73
|
+
const ResourceErrorSchema = Schema.Struct({
|
|
74
|
+
...ResourceErrorContextSchema.fields,
|
|
75
|
+
at: Schema.String
|
|
76
|
+
});
|
|
77
|
+
/** Complete persisted state assembled from independently stored components. */
|
|
78
|
+
const ResourceStateSchema = Schema.Struct({
|
|
79
|
+
backfill: Schema.optional(BackfillStateSchema),
|
|
80
|
+
changes: Schema.optional(ChangesStateSchema),
|
|
81
|
+
lastError: Schema.optional(ResourceErrorSchema)
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
//#endregion
|
|
85
|
+
//#region src/state-store/layer-key-value-store.ts
|
|
86
|
+
const mapStoreError = (operation, resource) => Effect.mapError((cause) => new ConnectorError({
|
|
87
|
+
message: `Failed to ${operation} state for resource ${resource}`,
|
|
88
|
+
cause
|
|
89
|
+
}));
|
|
90
|
+
const resourceErrorContext = (source, operation) => {
|
|
91
|
+
const sourceName = source === "backfill" ? "Backfill" : "Changes";
|
|
92
|
+
switch (operation) {
|
|
93
|
+
case "fetch": return {
|
|
94
|
+
source,
|
|
95
|
+
operation,
|
|
96
|
+
code: "fetch_failed",
|
|
97
|
+
message: `${sourceName} fetch failed`
|
|
98
|
+
};
|
|
99
|
+
case "publish": return {
|
|
100
|
+
source,
|
|
101
|
+
operation,
|
|
102
|
+
code: "publish_failed",
|
|
103
|
+
message: `${sourceName} publish failed`
|
|
104
|
+
};
|
|
105
|
+
case "checkpoint": return {
|
|
106
|
+
source,
|
|
107
|
+
operation,
|
|
108
|
+
code: "checkpoint_failed",
|
|
109
|
+
message: `${sourceName} checkpoint failed`
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
const latestError = (backfill, changes) => {
|
|
114
|
+
if (Option.isNone(backfill)) return changes;
|
|
115
|
+
if (Option.isNone(changes)) return backfill;
|
|
116
|
+
return backfill.value.at >= changes.value.at ? backfill : changes;
|
|
117
|
+
};
|
|
118
|
+
/**
|
|
119
|
+
* Implements StateStore over any Effect KeyValueStore. `toSchemaStore` owns
|
|
120
|
+
* serialization and validates persisted values when they are read and written.
|
|
121
|
+
* The caller owns instance scoping; `layerSql` supplies the production prefix.
|
|
122
|
+
*/
|
|
123
|
+
const layerKeyValueStore = Layer.effect(StateStore)(Effect.gen(function* () {
|
|
124
|
+
const keyValueStore = yield* KeyValueStore.KeyValueStore;
|
|
125
|
+
const backfillStore = KeyValueStore.toSchemaStore(keyValueStore, BackfillStateSchema);
|
|
126
|
+
const changesStore = KeyValueStore.toSchemaStore(keyValueStore, ChangesStateSchema);
|
|
127
|
+
const errorStore = KeyValueStore.toSchemaStore(keyValueStore, ResourceErrorSchema);
|
|
128
|
+
const getResourceState = (resource) => Effect.all({
|
|
129
|
+
backfill: backfillStore.get(resourceComponentKey(resource, "backfill")),
|
|
130
|
+
changes: changesStore.get(resourceComponentKey(resource, "changes")),
|
|
131
|
+
backfillError: errorStore.get(resourceErrorKey(resource, "backfill")),
|
|
132
|
+
changesError: errorStore.get(resourceErrorKey(resource, "changes"))
|
|
133
|
+
}, { concurrency: "unbounded" }).pipe(Effect.map(({ backfill, changes, backfillError, changesError }) => {
|
|
134
|
+
const lastError = latestError(backfillError, changesError);
|
|
135
|
+
if (Option.isNone(backfill) && Option.isNone(changes) && Option.isNone(lastError)) return;
|
|
136
|
+
return {
|
|
137
|
+
...Option.isSome(backfill) ? { backfill: backfill.value } : {},
|
|
138
|
+
...Option.isSome(changes) ? { changes: changes.value } : {},
|
|
139
|
+
...Option.isSome(lastError) ? { lastError: lastError.value } : {}
|
|
140
|
+
};
|
|
141
|
+
}), mapStoreError("read", resource));
|
|
142
|
+
const setBackfillState = (resource, state) => backfillStore.set(resourceComponentKey(resource, "backfill"), state).pipe(mapStoreError("write", resource));
|
|
143
|
+
const setChangesState = (resource, state) => changesStore.set(resourceComponentKey(resource, "changes"), state).pipe(mapStoreError("write", resource));
|
|
144
|
+
const setResourceError = (resource, source, operation) => DateTime.now.pipe(Effect.map(DateTime.formatIso), Effect.flatMap((at) => {
|
|
145
|
+
const error = resourceErrorContext(source, operation);
|
|
146
|
+
return errorStore.set(resourceErrorKey(resource, source), {
|
|
147
|
+
...error,
|
|
148
|
+
at
|
|
149
|
+
});
|
|
150
|
+
}), mapStoreError("write", resource));
|
|
151
|
+
const clearResourceError = (resource, source) => errorStore.remove(resourceErrorKey(resource, source)).pipe(mapStoreError("clear", resource));
|
|
152
|
+
return StateStore.of({
|
|
153
|
+
getResourceState,
|
|
154
|
+
setBackfillState,
|
|
155
|
+
setChangesState,
|
|
156
|
+
setResourceError,
|
|
157
|
+
clearResourceError
|
|
158
|
+
});
|
|
159
|
+
}));
|
|
160
|
+
|
|
161
|
+
//#endregion
|
|
162
|
+
//#region src/state-store/layer-memory.ts
|
|
163
|
+
/** Process-local state for tests and explicit local sandbox commands. */
|
|
164
|
+
const layerMemory = layerKeyValueStore.pipe(Layer.provide(KeyValueStore.layerMemory));
|
|
165
|
+
|
|
166
|
+
//#endregion
|
|
167
|
+
//#region src/state-store/layer-sql.ts
|
|
168
|
+
const scopedKeyValueStore = (connectorInstanceId) => Layer.effect(KeyValueStore.KeyValueStore)(KeyValueStore.KeyValueStore.pipe(Effect.map(KeyValueStore.prefix(connectorInstanceKeyPrefix(connectorInstanceId)))));
|
|
169
|
+
/**
|
|
170
|
+
* Shared SQL state using the immutable connector instance ID supplied by
|
|
171
|
+
* platform-owned Effect configuration. The table defaults to
|
|
172
|
+
* `_airfoil_connectors_state` and can be overridden with `AIRFOIL_STATE_TABLE`.
|
|
173
|
+
* Effect's SQL KeyValueStore owns the table, while `KeyValueStore.prefix`
|
|
174
|
+
* provides application-level isolation between connector instances in that
|
|
175
|
+
* table. The resulting layer exposes only StateStore, never the unscoped
|
|
176
|
+
* KeyValueStore.
|
|
177
|
+
*
|
|
178
|
+
* @example
|
|
179
|
+
* ```ts
|
|
180
|
+
* const StateStoreLive = layerSql().pipe(Layer.provide(PostgresLive));
|
|
181
|
+
* ```
|
|
182
|
+
*/
|
|
183
|
+
const layerSql = () => Layer.unwrap(Config.all({
|
|
184
|
+
connectorInstanceId: Config.nonEmptyString(PlatformRuntimeKey.connectorInstanceId),
|
|
185
|
+
table: Config.nonEmptyString(PlatformRuntimeKey.stateTable).pipe(Config.withDefault(PlatformRuntimeDefault.stateTable))
|
|
186
|
+
}).pipe(Effect.map(({ connectorInstanceId, table }) => layerKeyValueStore.pipe(Layer.provide(scopedKeyValueStore(connectorInstanceId).pipe(Layer.provide(KeyValueStore.layerSql({ table }))))))));
|
|
187
|
+
|
|
188
|
+
//#endregion
|
|
189
|
+
//#region src/state-store/index.ts
|
|
190
|
+
var state_store_exports = /* @__PURE__ */ __export({
|
|
191
|
+
StateStore: () => StateStore,
|
|
192
|
+
layerMemory: () => layerMemory,
|
|
193
|
+
layerSql: () => layerSql
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
//#endregion
|
|
197
|
+
export { ResourceStateSchema as a, ResourceErrorSchema as i, layerSql as n, StateStore as o, layerMemory as r, state_store_exports as t };
|
|
198
|
+
//# sourceMappingURL=state-store-yl6LiTQ0.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"state-store-yl6LiTQ0.js","names":["layerKeyValueStore: Layer.Layer<StateStore, never, KeyValueStore.KeyValueStore>","layerMemory: Layer.Layer<StateStore>"],"sources":["../src/state-store/service.ts","../src/state-store/keys.ts","../src/state-store/schema.ts","../src/state-store/layer-key-value-store.ts","../src/state-store/layer-memory.ts","../src/state-store/layer-sql.ts","../src/state-store/index.ts"],"sourcesContent":["import { Context, Effect } from \"effect\";\n\nimport type {\n PersistedBackfillState,\n PersistedChangesState,\n PersistedResourceState,\n StateOperation,\n StateSource,\n} from \"./schema\";\n\nimport { ConnectorError } from \"../errors\";\n\n/** Connector-facing durable state operations, independent of the storage backend. */\nexport interface StateStoreService {\n readonly getResourceState: (\n resource: string,\n ) => Effect.Effect<PersistedResourceState | undefined, ConnectorError>;\n readonly setBackfillState: (\n resource: string,\n state: PersistedBackfillState,\n ) => Effect.Effect<void, ConnectorError>;\n readonly setChangesState: (\n resource: string,\n state: PersistedChangesState,\n ) => Effect.Effect<void, ConnectorError>;\n readonly setResourceError: (\n resource: string,\n source: StateSource,\n operation: StateOperation,\n ) => Effect.Effect<void, ConnectorError>;\n readonly clearResourceError: (\n resource: string,\n source: StateSource,\n ) => Effect.Effect<void, ConnectorError>;\n}\n\n/**\n * Effect service used by ingestion and status logic for connector checkpoints.\n * Production provides `layerSql`; sandboxes and tests may provide `layerMemory`.\n *\n * @example\n * ```ts\n * const program = Effect.gen(function* () {\n * const state = yield* StateStore;\n * yield* state.setChangesState(\"orders\", { cursor: \"next-page\" });\n * });\n * ```\n */\nexport class StateStore extends Context.Service<StateStore, StateStoreService>()(\n \"@useairfoil/connector-kit/StateStore\",\n) {}\n","import type { StateSource } from \"./schema\";\n\ntype StateComponent = StateSource | `error:${StateSource}`;\n\nexport const resourceComponentKey = (resource: string, component: StateComponent): string =>\n `resource:${resource}:${component}`;\n\nexport const resourceErrorKey = (resource: string, source: StateSource): string =>\n resourceComponentKey(resource, `error:${source}`);\n\n/**\n * Prefix for all state owned by one connector instance. The platform-owned ID\n * stays visible so operators can inspect and clean up shared-table rows directly.\n *\n * @example\n * ```ts\n * connectorInstanceKeyPrefix(\"team-a/orders\");\n * // \"connector-instance:team-a/orders:\"\n * ```\n */\nexport const connectorInstanceKeyPrefix = (connectorInstanceId: string): string =>\n `connector-instance:${connectorInstanceId}:`;\n","import { Schema } from \"effect\";\n\nconst CursorValueSchema = Schema.Union([Schema.String, Schema.Number]);\n\n/** Durable progress for a finite historical backfill. */\nexport const BackfillStateSchema = Schema.Struct({\n cutoff: CursorValueSchema,\n pageCursor: Schema.optional(CursorValueSchema),\n completed: Schema.Boolean,\n lastSuccessAt: Schema.optional(Schema.String),\n});\n\n/** Durable cursor for an ongoing incremental changes stream. */\nexport const ChangesStateSchema = Schema.Struct({\n cursor: CursorValueSchema,\n lastSuccessAt: Schema.optional(Schema.String),\n});\n\nexport const StateSourceSchema = Schema.Literals([\"backfill\", \"changes\"]);\n\nexport const StateOperationSchema = Schema.Literals([\"fetch\", \"publish\", \"checkpoint\"]);\n\nexport const StateErrorCodeSchema = Schema.Literals([\n \"fetch_failed\",\n \"publish_failed\",\n \"checkpoint_failed\",\n]);\n\n/** SDK-owned error context safe to persist and expose through status. */\nexport const ResourceErrorContextSchema = Schema.Struct({\n source: StateSourceSchema,\n operation: StateOperationSchema,\n code: StateErrorCodeSchema,\n message: Schema.String,\n});\n\n/** Stable public error summary persisted for status reporting; never a raw cause. */\nexport const ResourceErrorSchema = Schema.Struct({\n ...ResourceErrorContextSchema.fields,\n at: Schema.String,\n});\n\n/** Complete persisted state assembled from independently stored components. */\nexport const ResourceStateSchema = Schema.Struct({\n backfill: Schema.optional(BackfillStateSchema),\n changes: Schema.optional(ChangesStateSchema),\n lastError: Schema.optional(ResourceErrorSchema),\n});\n\nexport type PersistedBackfillState = Schema.Schema.Type<typeof BackfillStateSchema>;\nexport type PersistedChangesState = Schema.Schema.Type<typeof ChangesStateSchema>;\nexport type StateSource = Schema.Schema.Type<typeof StateSourceSchema>;\nexport type StateOperation = Schema.Schema.Type<typeof StateOperationSchema>;\nexport type StateErrorCode = Schema.Schema.Type<typeof StateErrorCodeSchema>;\nexport type ResourceErrorContext = Schema.Schema.Type<typeof ResourceErrorContextSchema>;\nexport type PersistedResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;\nexport type PersistedResourceState = Schema.Schema.Type<typeof ResourceStateSchema>;\n","import { DateTime, Effect, Layer, Option, Schema } from \"effect\";\nimport { KeyValueStore } from \"effect/unstable/persistence\";\n\nimport { ConnectorError } from \"../errors\";\nimport { resourceComponentKey, resourceErrorKey } from \"./keys\";\nimport {\n BackfillStateSchema,\n ChangesStateSchema,\n ResourceErrorSchema,\n type PersistedBackfillState,\n type PersistedChangesState,\n type PersistedResourceError,\n type ResourceErrorContext,\n type StateOperation,\n type StateSource,\n} from \"./schema\";\nimport { StateStore } from \"./service\";\n\nconst mapStoreError = (operation: \"read\" | \"write\" | \"clear\", resource: string) =>\n Effect.mapError(\n (cause: KeyValueStore.KeyValueStoreError | Schema.SchemaError) =>\n new ConnectorError({\n message: `Failed to ${operation} state for resource ${resource}`,\n cause,\n }),\n );\n\nconst resourceErrorContext = (\n source: StateSource,\n operation: StateOperation,\n): ResourceErrorContext => {\n const sourceName = source === \"backfill\" ? \"Backfill\" : \"Changes\";\n switch (operation) {\n case \"fetch\":\n return { source, operation, code: \"fetch_failed\", message: `${sourceName} fetch failed` };\n case \"publish\":\n return { source, operation, code: \"publish_failed\", message: `${sourceName} publish failed` };\n case \"checkpoint\":\n return {\n source,\n operation,\n code: \"checkpoint_failed\",\n message: `${sourceName} checkpoint failed`,\n };\n }\n};\n\nconst latestError = (\n backfill: Option.Option<PersistedResourceError>,\n changes: Option.Option<PersistedResourceError>,\n): Option.Option<PersistedResourceError> => {\n if (Option.isNone(backfill)) return changes;\n if (Option.isNone(changes)) return backfill;\n return backfill.value.at >= changes.value.at ? backfill : changes;\n};\n\n/**\n * Implements StateStore over any Effect KeyValueStore. `toSchemaStore` owns\n * serialization and validates persisted values when they are read and written.\n * The caller owns instance scoping; `layerSql` supplies the production prefix.\n */\nexport const layerKeyValueStore: Layer.Layer<StateStore, never, KeyValueStore.KeyValueStore> =\n Layer.effect(StateStore)(\n Effect.gen(function* () {\n const keyValueStore = yield* KeyValueStore.KeyValueStore;\n const backfillStore = KeyValueStore.toSchemaStore(keyValueStore, BackfillStateSchema);\n const changesStore = KeyValueStore.toSchemaStore(keyValueStore, ChangesStateSchema);\n const errorStore = KeyValueStore.toSchemaStore(keyValueStore, ResourceErrorSchema);\n\n const getResourceState = (resource: string) =>\n // Components use separate keys so a cursor update cannot overwrite the\n // independently managed error or backfill state.\n Effect.all(\n {\n backfill: backfillStore.get(resourceComponentKey(resource, \"backfill\")),\n changes: changesStore.get(resourceComponentKey(resource, \"changes\")),\n backfillError: errorStore.get(resourceErrorKey(resource, \"backfill\")),\n changesError: errorStore.get(resourceErrorKey(resource, \"changes\")),\n },\n { concurrency: \"unbounded\" },\n ).pipe(\n Effect.map(({ backfill, changes, backfillError, changesError }) => {\n const lastError = latestError(backfillError, changesError);\n if (Option.isNone(backfill) && Option.isNone(changes) && Option.isNone(lastError)) {\n return undefined;\n }\n return {\n ...(Option.isSome(backfill) ? { backfill: backfill.value } : {}),\n ...(Option.isSome(changes) ? { changes: changes.value } : {}),\n ...(Option.isSome(lastError) ? { lastError: lastError.value } : {}),\n };\n }),\n mapStoreError(\"read\", resource),\n );\n\n const setBackfillState = (resource: string, state: PersistedBackfillState) =>\n backfillStore\n .set(resourceComponentKey(resource, \"backfill\"), state)\n .pipe(mapStoreError(\"write\", resource));\n\n const setChangesState = (resource: string, state: PersistedChangesState) =>\n changesStore\n .set(resourceComponentKey(resource, \"changes\"), state)\n .pipe(mapStoreError(\"write\", resource));\n\n const setResourceError = (resource: string, source: StateSource, operation: StateOperation) =>\n DateTime.now.pipe(\n Effect.map(DateTime.formatIso),\n Effect.flatMap((at) => {\n const error = resourceErrorContext(source, operation);\n return errorStore.set(resourceErrorKey(resource, source), {\n ...error,\n at,\n });\n }),\n mapStoreError(\"write\", resource),\n );\n\n const clearResourceError = (resource: string, source: StateSource) =>\n errorStore\n .remove(resourceErrorKey(resource, source))\n .pipe(mapStoreError(\"clear\", resource));\n\n return StateStore.of({\n getResourceState,\n setBackfillState,\n setChangesState,\n setResourceError,\n clearResourceError,\n });\n }),\n );\n","import { Layer } from \"effect\";\nimport { KeyValueStore } from \"effect/unstable/persistence\";\n\nimport type { StateStore } from \"./service\";\n\nimport { layerKeyValueStore } from \"./layer-key-value-store\";\n\n/** Process-local state for tests and explicit local sandbox commands. */\nexport const layerMemory: Layer.Layer<StateStore> = layerKeyValueStore.pipe(\n Layer.provide(KeyValueStore.layerMemory),\n);\n","import { Config, Effect, Layer } from \"effect\";\nimport { KeyValueStore } from \"effect/unstable/persistence\";\nimport { SqlClient } from \"effect/unstable/sql\";\n\nimport type { StateStore } from \"./service\";\n\nimport { PlatformRuntimeDefault, PlatformRuntimeKey } from \"../runtime-config/constants\";\nimport { connectorInstanceKeyPrefix } from \"./keys\";\nimport { layerKeyValueStore } from \"./layer-key-value-store\";\n\nconst scopedKeyValueStore = (connectorInstanceId: string) =>\n Layer.effect(KeyValueStore.KeyValueStore)(\n KeyValueStore.KeyValueStore.pipe(\n Effect.map(KeyValueStore.prefix(connectorInstanceKeyPrefix(connectorInstanceId))),\n ),\n );\n\n/**\n * Shared SQL state using the immutable connector instance ID supplied by\n * platform-owned Effect configuration. The table defaults to\n * `_airfoil_connectors_state` and can be overridden with `AIRFOIL_STATE_TABLE`.\n * Effect's SQL KeyValueStore owns the table, while `KeyValueStore.prefix`\n * provides application-level isolation between connector instances in that\n * table. The resulting layer exposes only StateStore, never the unscoped\n * KeyValueStore.\n *\n * @example\n * ```ts\n * const StateStoreLive = layerSql().pipe(Layer.provide(PostgresLive));\n * ```\n */\nexport const layerSql = (): Layer.Layer<StateStore, Config.ConfigError, SqlClient.SqlClient> =>\n Layer.unwrap(\n Config.all({\n connectorInstanceId: Config.nonEmptyString(PlatformRuntimeKey.connectorInstanceId),\n table: Config.nonEmptyString(PlatformRuntimeKey.stateTable).pipe(\n Config.withDefault(PlatformRuntimeDefault.stateTable),\n ),\n }).pipe(\n Effect.map(({ connectorInstanceId, table }) =>\n layerKeyValueStore.pipe(\n Layer.provide(\n scopedKeyValueStore(connectorInstanceId).pipe(\n Layer.provide(KeyValueStore.layerSql({ table })),\n ),\n ),\n ),\n ),\n ),\n );\n","export { StateStore, type StateStoreService } from \"./service\";\nexport { layerMemory } from \"./layer-memory\";\nexport { layerSql } from \"./layer-sql\";\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAgDA,IAAa,aAAb,cAAgC,QAAQ,SAAwC,CAC9E,uCACD,CAAC;;;;AC9CF,MAAa,wBAAwB,UAAkB,cACrD,YAAY,SAAS,GAAG;AAE1B,MAAa,oBAAoB,UAAkB,WACjD,qBAAqB,UAAU,SAAS,SAAS;;;;;;;;;;;AAYnD,MAAa,8BAA8B,wBACzC,sBAAsB,oBAAoB;;;;ACnB5C,MAAM,oBAAoB,OAAO,MAAM,CAAC,OAAO,QAAQ,OAAO,OAAO,CAAC;;AAGtE,MAAa,sBAAsB,OAAO,OAAO;CAC/C,QAAQ;CACR,YAAY,OAAO,SAAS,kBAAkB;CAC9C,WAAW,OAAO;CAClB,eAAe,OAAO,SAAS,OAAO,OAAO;CAC9C,CAAC;;AAGF,MAAa,qBAAqB,OAAO,OAAO;CAC9C,QAAQ;CACR,eAAe,OAAO,SAAS,OAAO,OAAO;CAC9C,CAAC;AAEF,MAAa,oBAAoB,OAAO,SAAS,CAAC,YAAY,UAAU,CAAC;AAEzE,MAAa,uBAAuB,OAAO,SAAS;CAAC;CAAS;CAAW;CAAa,CAAC;AAEvF,MAAa,uBAAuB,OAAO,SAAS;CAClD;CACA;CACA;CACD,CAAC;;AAGF,MAAa,6BAA6B,OAAO,OAAO;CACtD,QAAQ;CACR,WAAW;CACX,MAAM;CACN,SAAS,OAAO;CACjB,CAAC;;AAGF,MAAa,sBAAsB,OAAO,OAAO;CAC/C,GAAG,2BAA2B;CAC9B,IAAI,OAAO;CACZ,CAAC;;AAGF,MAAa,sBAAsB,OAAO,OAAO;CAC/C,UAAU,OAAO,SAAS,oBAAoB;CAC9C,SAAS,OAAO,SAAS,mBAAmB;CAC5C,WAAW,OAAO,SAAS,oBAAoB;CAChD,CAAC;;;;AC7BF,MAAM,iBAAiB,WAAuC,aAC5D,OAAO,UACJ,UACC,IAAI,eAAe;CACjB,SAAS,aAAa,UAAU,sBAAsB;CACtD;CACD,CAAC,CACL;AAEH,MAAM,wBACJ,QACA,cACyB;CACzB,MAAM,aAAa,WAAW,aAAa,aAAa;AACxD,SAAQ,WAAR;EACE,KAAK,QACH,QAAO;GAAE;GAAQ;GAAW,MAAM;GAAgB,SAAS,GAAG,WAAW;GAAgB;EAC3F,KAAK,UACH,QAAO;GAAE;GAAQ;GAAW,MAAM;GAAkB,SAAS,GAAG,WAAW;GAAkB;EAC/F,KAAK,aACH,QAAO;GACL;GACA;GACA,MAAM;GACN,SAAS,GAAG,WAAW;GACxB;;;AAIP,MAAM,eACJ,UACA,YAC0C;AAC1C,KAAI,OAAO,OAAO,SAAS,CAAE,QAAO;AACpC,KAAI,OAAO,OAAO,QAAQ,CAAE,QAAO;AACnC,QAAO,SAAS,MAAM,MAAM,QAAQ,MAAM,KAAK,WAAW;;;;;;;AAQ5D,MAAaA,qBACX,MAAM,OAAO,WAAW,CACtB,OAAO,IAAI,aAAa;CACtB,MAAM,gBAAgB,OAAO,cAAc;CAC3C,MAAM,gBAAgB,cAAc,cAAc,eAAe,oBAAoB;CACrF,MAAM,eAAe,cAAc,cAAc,eAAe,mBAAmB;CACnF,MAAM,aAAa,cAAc,cAAc,eAAe,oBAAoB;CAElF,MAAM,oBAAoB,aAGxB,OAAO,IACL;EACE,UAAU,cAAc,IAAI,qBAAqB,UAAU,WAAW,CAAC;EACvE,SAAS,aAAa,IAAI,qBAAqB,UAAU,UAAU,CAAC;EACpE,eAAe,WAAW,IAAI,iBAAiB,UAAU,WAAW,CAAC;EACrE,cAAc,WAAW,IAAI,iBAAiB,UAAU,UAAU,CAAC;EACpE,EACD,EAAE,aAAa,aAAa,CAC7B,CAAC,KACA,OAAO,KAAK,EAAE,UAAU,SAAS,eAAe,mBAAmB;EACjE,MAAM,YAAY,YAAY,eAAe,aAAa;AAC1D,MAAI,OAAO,OAAO,SAAS,IAAI,OAAO,OAAO,QAAQ,IAAI,OAAO,OAAO,UAAU,CAC/E;AAEF,SAAO;GACL,GAAI,OAAO,OAAO,SAAS,GAAG,EAAE,UAAU,SAAS,OAAO,GAAG,EAAE;GAC/D,GAAI,OAAO,OAAO,QAAQ,GAAG,EAAE,SAAS,QAAQ,OAAO,GAAG,EAAE;GAC5D,GAAI,OAAO,OAAO,UAAU,GAAG,EAAE,WAAW,UAAU,OAAO,GAAG,EAAE;GACnE;GACD,EACF,cAAc,QAAQ,SAAS,CAChC;CAEH,MAAM,oBAAoB,UAAkB,UAC1C,cACG,IAAI,qBAAqB,UAAU,WAAW,EAAE,MAAM,CACtD,KAAK,cAAc,SAAS,SAAS,CAAC;CAE3C,MAAM,mBAAmB,UAAkB,UACzC,aACG,IAAI,qBAAqB,UAAU,UAAU,EAAE,MAAM,CACrD,KAAK,cAAc,SAAS,SAAS,CAAC;CAE3C,MAAM,oBAAoB,UAAkB,QAAqB,cAC/D,SAAS,IAAI,KACX,OAAO,IAAI,SAAS,UAAU,EAC9B,OAAO,SAAS,OAAO;EACrB,MAAM,QAAQ,qBAAqB,QAAQ,UAAU;AACrD,SAAO,WAAW,IAAI,iBAAiB,UAAU,OAAO,EAAE;GACxD,GAAG;GACH;GACD,CAAC;GACF,EACF,cAAc,SAAS,SAAS,CACjC;CAEH,MAAM,sBAAsB,UAAkB,WAC5C,WACG,OAAO,iBAAiB,UAAU,OAAO,CAAC,CAC1C,KAAK,cAAc,SAAS,SAAS,CAAC;AAE3C,QAAO,WAAW,GAAG;EACnB;EACA;EACA;EACA;EACA;EACD,CAAC;EACF,CACH;;;;;AC3HH,MAAaC,cAAuC,mBAAmB,KACrE,MAAM,QAAQ,cAAc,YAAY,CACzC;;;;ACAD,MAAM,uBAAuB,wBAC3B,MAAM,OAAO,cAAc,cAAc,CACvC,cAAc,cAAc,KAC1B,OAAO,IAAI,cAAc,OAAO,2BAA2B,oBAAoB,CAAC,CAAC,CAClF,CACF;;;;;;;;;;;;;;;AAgBH,MAAa,iBACX,MAAM,OACJ,OAAO,IAAI;CACT,qBAAqB,OAAO,eAAe,mBAAmB,oBAAoB;CAClF,OAAO,OAAO,eAAe,mBAAmB,WAAW,CAAC,KAC1D,OAAO,YAAY,uBAAuB,WAAW,CACtD;CACF,CAAC,CAAC,KACD,OAAO,KAAK,EAAE,qBAAqB,YACjC,mBAAmB,KACjB,MAAM,QACJ,oBAAoB,oBAAoB,CAAC,KACvC,MAAM,QAAQ,cAAc,SAAS,EAAE,OAAO,CAAC,CAAC,CACjD,CACF,CACF,CACF,CACF,CACF"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import "../index-D54ySAKn.js";
|
|
2
|
+
import "../types-B35J5YPu.js";
|
|
3
|
+
import "../index-Coh_SkBs.js";
|
|
4
|
+
import { a as ResourceStatusSchema, c as StatusUnavailableResponse, d as SyncStateSchema, f as statusRoute, i as ResourceStatus, l as StatusUnavailableResponseSchema, n as ResourceError, o as StatusResponse, r as ResourceErrorSchema, s as StatusResponseSchema, u as SyncState } from "../index-CTLYQ74P.js";
|
|
5
|
+
export { ResourceError, ResourceErrorSchema, ResourceStatus, ResourceStatusSchema, StatusResponse, StatusResponseSchema, StatusUnavailableResponse, StatusUnavailableResponseSchema, SyncState, SyncStateSchema, statusRoute };
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import "../errors-DHWylqRZ.js";
|
|
2
|
+
import "../constants-BRhIX_rz.js";
|
|
3
|
+
import "../state-store-yl6LiTQ0.js";
|
|
4
|
+
import { a as StatusUnavailableResponseSchema, i as StatusResponseSchema, n as ResourceErrorSchema, o as SyncStateSchema, r as ResourceStatusSchema, s as statusRoute } from "../status-B-oZBc33.js";
|
|
5
|
+
|
|
6
|
+
export { ResourceErrorSchema, ResourceStatusSchema, StatusResponseSchema, StatusUnavailableResponseSchema, SyncStateSchema, statusRoute };
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { a as ResourceStateSchema, i as ResourceErrorSchema, o as StateStore } from "./state-store-yl6LiTQ0.js";
|
|
3
|
+
import { Cause, Effect, Schema } from "effect";
|
|
4
|
+
import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
|
|
5
|
+
|
|
6
|
+
//#region src/state-store/state.ts
|
|
7
|
+
/** Normalizes a cursor value to its persisted/wire representation (never a Date). */
|
|
8
|
+
const normalizeCursor = (value) => value instanceof Date ? value.toISOString() : value;
|
|
9
|
+
/** Derives connector-facing sync state from durable state and resource capabilities. */
|
|
10
|
+
const deriveSyncState = (resource, state) => {
|
|
11
|
+
if (state?.lastError) return "error";
|
|
12
|
+
if (!resource.backfill && !resource.changes) return "live";
|
|
13
|
+
if (!state) return "pending";
|
|
14
|
+
if (resource.backfill && (!state.backfill || !state.backfill.completed)) return "backfilling";
|
|
15
|
+
return "live";
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
//#endregion
|
|
19
|
+
//#region src/status/route.ts
|
|
20
|
+
const statusRoute = (connector, path = "/status") => HttpRouter.add("GET", path, Effect.gen(function* () {
|
|
21
|
+
const store = yield* StateStore;
|
|
22
|
+
const resources = yield* Effect.forEach(connector.resources, (resource) => store.getResourceState(resource.name).pipe(Effect.map((state) => ({
|
|
23
|
+
name: resource.name,
|
|
24
|
+
state: deriveSyncState(resource, state),
|
|
25
|
+
...state?.backfill ? { backfill: {
|
|
26
|
+
completed: state.backfill.completed,
|
|
27
|
+
cutoff: normalizeCursor(state.backfill.cutoff),
|
|
28
|
+
...state.backfill.pageCursor !== void 0 ? { pageCursor: normalizeCursor(state.backfill.pageCursor) } : {},
|
|
29
|
+
...state.backfill.lastSuccessAt !== void 0 ? { lastSuccessAt: state.backfill.lastSuccessAt } : {}
|
|
30
|
+
} } : {},
|
|
31
|
+
...state?.changes ? { changes: {
|
|
32
|
+
cursor: normalizeCursor(state.changes.cursor),
|
|
33
|
+
...state.changes.lastSuccessAt !== void 0 ? { lastSuccessAt: state.changes.lastSuccessAt } : {}
|
|
34
|
+
} } : {},
|
|
35
|
+
...state?.lastError ? { lastError: state.lastError } : {}
|
|
36
|
+
}))), { concurrency: "unbounded" });
|
|
37
|
+
return HttpServerResponse.jsonUnsafe({
|
|
38
|
+
connector: connector.name,
|
|
39
|
+
resources
|
|
40
|
+
});
|
|
41
|
+
}).pipe(Effect.catchCause((cause) => Effect.logWarning(`Status route failed: ${Cause.pretty(cause)}`).pipe(Effect.andThen(Effect.succeed(HttpServerResponse.jsonUnsafe({
|
|
42
|
+
ok: false,
|
|
43
|
+
error: "Failed to compute status"
|
|
44
|
+
}, { status: 500 })))))));
|
|
45
|
+
|
|
46
|
+
//#endregion
|
|
47
|
+
//#region src/status/types.ts
|
|
48
|
+
const SyncStateSchema = Schema.Literals([
|
|
49
|
+
"pending",
|
|
50
|
+
"backfilling",
|
|
51
|
+
"live",
|
|
52
|
+
"error"
|
|
53
|
+
]);
|
|
54
|
+
/** Safe source failure returned by the status endpoint. */
|
|
55
|
+
const ResourceErrorSchema$1 = ResourceErrorSchema;
|
|
56
|
+
const ResourceStatusSchema = Schema.Struct({
|
|
57
|
+
name: Schema.String,
|
|
58
|
+
state: SyncStateSchema,
|
|
59
|
+
...ResourceStateSchema.fields
|
|
60
|
+
});
|
|
61
|
+
const StatusResponseSchema = Schema.Struct({
|
|
62
|
+
connector: Schema.String,
|
|
63
|
+
resources: Schema.Array(ResourceStatusSchema)
|
|
64
|
+
});
|
|
65
|
+
/** Returned when durable resource status cannot be read. */
|
|
66
|
+
const StatusUnavailableResponseSchema = Schema.Struct({
|
|
67
|
+
ok: Schema.Literal(false),
|
|
68
|
+
error: Schema.String
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
//#endregion
|
|
72
|
+
//#region src/status/index.ts
|
|
73
|
+
var status_exports = /* @__PURE__ */ __export({
|
|
74
|
+
ResourceErrorSchema: () => ResourceErrorSchema$1,
|
|
75
|
+
ResourceStatusSchema: () => ResourceStatusSchema,
|
|
76
|
+
StatusResponseSchema: () => StatusResponseSchema,
|
|
77
|
+
StatusUnavailableResponseSchema: () => StatusUnavailableResponseSchema,
|
|
78
|
+
SyncStateSchema: () => SyncStateSchema,
|
|
79
|
+
statusRoute: () => statusRoute
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
//#endregion
|
|
83
|
+
export { StatusUnavailableResponseSchema as a, deriveSyncState as c, StatusResponseSchema as i, normalizeCursor as l, ResourceErrorSchema$1 as n, SyncStateSchema as o, ResourceStatusSchema as r, statusRoute as s, status_exports as t };
|
|
84
|
+
//# sourceMappingURL=status-B-oZBc33.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"status-B-oZBc33.js","names":["ResourceErrorSchema","StateResourceErrorSchema"],"sources":["../src/state-store/state.ts","../src/status/route.ts","../src/status/types.ts","../src/status/index.ts"],"sourcesContent":["import type { Cursor, ResourceDefinition, ResourceState, SyncState } from \"../core/types\";\n\n/** Normalizes a cursor value to its persisted/wire representation (never a Date). */\nexport const normalizeCursor = (value: Cursor.Value): string | number =>\n value instanceof Date ? value.toISOString() : value;\n\n/** Derives connector-facing sync state from durable state and resource capabilities. */\nexport const deriveSyncState = (\n resource: ResourceDefinition,\n state: ResourceState | undefined,\n): SyncState => {\n if (state?.lastError) return \"error\";\n if (!resource.backfill && !resource.changes) return \"live\";\n if (!state) return \"pending\";\n if (resource.backfill && (!state.backfill || !state.backfill.completed)) return \"backfilling\";\n return \"live\";\n};\n","import { Cause, Effect } from \"effect\";\nimport { HttpRouter, HttpServerResponse } from \"effect/unstable/http\";\n\nimport type { ConnectorDefinition } from \"../core/types\";\nimport type { ResourceStatus, StatusUnavailableResponse } from \"./types\";\n\nimport { StateStore } from \"../state-store\";\nimport { deriveSyncState, normalizeCursor } from \"../state-store/state\";\n\nexport const statusRoute = (\n connector: ConnectorDefinition,\n path: HttpRouter.PathInput = \"/status\",\n) =>\n HttpRouter.add(\n \"GET\",\n path,\n Effect.gen(function* () {\n const store = yield* StateStore;\n\n const resources = yield* Effect.forEach(\n connector.resources,\n (resource) =>\n store.getResourceState(resource.name).pipe(\n Effect.map(\n (state): ResourceStatus => ({\n name: resource.name,\n state: deriveSyncState(resource, state),\n ...(state?.backfill\n ? {\n backfill: {\n completed: state.backfill.completed,\n cutoff: normalizeCursor(state.backfill.cutoff),\n ...(state.backfill.pageCursor !== undefined\n ? { pageCursor: normalizeCursor(state.backfill.pageCursor) }\n : {}),\n ...(state.backfill.lastSuccessAt !== undefined\n ? { lastSuccessAt: state.backfill.lastSuccessAt }\n : {}),\n },\n }\n : {}),\n ...(state?.changes\n ? {\n changes: {\n cursor: normalizeCursor(state.changes.cursor),\n ...(state.changes.lastSuccessAt !== undefined\n ? { lastSuccessAt: state.changes.lastSuccessAt }\n : {}),\n },\n }\n : {}),\n ...(state?.lastError ? { lastError: state.lastError } : {}),\n }),\n ),\n ),\n { concurrency: \"unbounded\" },\n );\n\n return HttpServerResponse.jsonUnsafe({ connector: connector.name, resources });\n }).pipe(\n Effect.catchCause((cause) =>\n Effect.logWarning(`Status route failed: ${Cause.pretty(cause)}`).pipe(\n Effect.andThen(\n Effect.succeed(\n HttpServerResponse.jsonUnsafe(\n {\n ok: false,\n error: \"Failed to compute status\",\n } satisfies StatusUnavailableResponse,\n { status: 500 },\n ),\n ),\n ),\n ),\n ),\n ),\n );\n","import { Schema } from \"effect\";\n\nimport {\n ResourceErrorSchema as StateResourceErrorSchema,\n ResourceStateSchema,\n} from \"../state-store/schema\";\n\nexport const SyncStateSchema = Schema.Literals([\"pending\", \"backfilling\", \"live\", \"error\"]);\nexport type SyncState = Schema.Schema.Type<typeof SyncStateSchema>;\n\n/** Safe source failure returned by the status endpoint. */\nexport const ResourceErrorSchema = StateResourceErrorSchema;\nexport type ResourceError = Schema.Schema.Type<typeof ResourceErrorSchema>;\n\nexport const ResourceStatusSchema = Schema.Struct({\n name: Schema.String,\n state: SyncStateSchema,\n ...ResourceStateSchema.fields,\n});\nexport type ResourceStatus = Schema.Schema.Type<typeof ResourceStatusSchema>;\n\nexport const StatusResponseSchema = Schema.Struct({\n connector: Schema.String,\n resources: Schema.Array(ResourceStatusSchema),\n});\nexport type StatusResponse = Schema.Schema.Type<typeof StatusResponseSchema>;\n\n/** Returned when durable resource status cannot be read. */\nexport const StatusUnavailableResponseSchema = Schema.Struct({\n ok: Schema.Literal(false),\n error: Schema.String,\n});\nexport type StatusUnavailableResponse = Schema.Schema.Type<typeof StatusUnavailableResponseSchema>;\n","export * from \"./route\";\nexport * from \"./types\";\n"],"mappings":";;;;;;;AAGA,MAAa,mBAAmB,UAC9B,iBAAiB,OAAO,MAAM,aAAa,GAAG;;AAGhD,MAAa,mBACX,UACA,UACc;AACd,KAAI,OAAO,UAAW,QAAO;AAC7B,KAAI,CAAC,SAAS,YAAY,CAAC,SAAS,QAAS,QAAO;AACpD,KAAI,CAAC,MAAO,QAAO;AACnB,KAAI,SAAS,aAAa,CAAC,MAAM,YAAY,CAAC,MAAM,SAAS,WAAY,QAAO;AAChF,QAAO;;;;;ACNT,MAAa,eACX,WACA,OAA6B,cAE7B,WAAW,IACT,OACA,MACA,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CAErB,MAAM,YAAY,OAAO,OAAO,QAC9B,UAAU,YACT,aACC,MAAM,iBAAiB,SAAS,KAAK,CAAC,KACpC,OAAO,KACJ,WAA2B;EAC1B,MAAM,SAAS;EACf,OAAO,gBAAgB,UAAU,MAAM;EACvC,GAAI,OAAO,WACP,EACE,UAAU;GACR,WAAW,MAAM,SAAS;GAC1B,QAAQ,gBAAgB,MAAM,SAAS,OAAO;GAC9C,GAAI,MAAM,SAAS,eAAe,SAC9B,EAAE,YAAY,gBAAgB,MAAM,SAAS,WAAW,EAAE,GAC1D,EAAE;GACN,GAAI,MAAM,SAAS,kBAAkB,SACjC,EAAE,eAAe,MAAM,SAAS,eAAe,GAC/C,EAAE;GACP,EACF,GACD,EAAE;EACN,GAAI,OAAO,UACP,EACE,SAAS;GACP,QAAQ,gBAAgB,MAAM,QAAQ,OAAO;GAC7C,GAAI,MAAM,QAAQ,kBAAkB,SAChC,EAAE,eAAe,MAAM,QAAQ,eAAe,GAC9C,EAAE;GACP,EACF,GACD,EAAE;EACN,GAAI,OAAO,YAAY,EAAE,WAAW,MAAM,WAAW,GAAG,EAAE;EAC3D,EACF,CACF,EACH,EAAE,aAAa,aAAa,CAC7B;AAED,QAAO,mBAAmB,WAAW;EAAE,WAAW,UAAU;EAAM;EAAW,CAAC;EAC9E,CAAC,KACD,OAAO,YAAY,UACjB,OAAO,WAAW,wBAAwB,MAAM,OAAO,MAAM,GAAG,CAAC,KAC/D,OAAO,QACL,OAAO,QACL,mBAAmB,WACjB;CACE,IAAI;CACJ,OAAO;CACR,EACD,EAAE,QAAQ,KAAK,CAChB,CACF,CACF,CACF,CACF,CACF,CACF;;;;ACrEH,MAAa,kBAAkB,OAAO,SAAS;CAAC;CAAW;CAAe;CAAQ;CAAQ,CAAC;;AAI3F,MAAaA,wBAAsBC;AAGnC,MAAa,uBAAuB,OAAO,OAAO;CAChD,MAAM,OAAO;CACb,OAAO;CACP,GAAG,oBAAoB;CACxB,CAAC;AAGF,MAAa,uBAAuB,OAAO,OAAO;CAChD,WAAW,OAAO;CAClB,WAAW,OAAO,MAAM,qBAAqB;CAC9C,CAAC;;AAIF,MAAa,kCAAkC,OAAO,OAAO;CAC3D,IAAI,OAAO,QAAQ,MAAM;CACzB,OAAO,OAAO;CACf,CAAC"}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { t as ConnectorError } from "./index-D54ySAKn.js";
|
|
2
|
+
import { Duration, Effect, Schema } from "effect";
|
|
3
|
+
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
|
|
4
|
+
|
|
5
|
+
//#region src/core/types.d.ts
|
|
6
|
+
declare namespace Cursor {
|
|
7
|
+
type Value = string | number | Date;
|
|
8
|
+
type Kind = "string" | "number" | "isoDateTime";
|
|
9
|
+
type Definition<T extends Value = Value> = {
|
|
10
|
+
readonly kind: Kind;
|
|
11
|
+
readonly decode: (value: unknown) => Effect.Effect<T, ConnectorError>;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
type DeleteValue = string | number | boolean | Date;
|
|
15
|
+
type ResourceMutation<Row extends object = object> = {
|
|
16
|
+
readonly op: "upsert";
|
|
17
|
+
readonly row: Row;
|
|
18
|
+
} | {
|
|
19
|
+
readonly op: "delete";
|
|
20
|
+
readonly key: DeleteValue;
|
|
21
|
+
readonly version: DeleteValue;
|
|
22
|
+
};
|
|
23
|
+
type ResourceBatch<Row extends object = object> = {
|
|
24
|
+
readonly cursor?: Cursor.Value;
|
|
25
|
+
readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
|
|
26
|
+
};
|
|
27
|
+
type ResourceState = {
|
|
28
|
+
readonly backfill?: {
|
|
29
|
+
readonly cutoff: Cursor.Value;
|
|
30
|
+
readonly pageCursor?: Cursor.Value;
|
|
31
|
+
readonly completed: boolean;
|
|
32
|
+
readonly lastSuccessAt?: string;
|
|
33
|
+
};
|
|
34
|
+
readonly changes?: {
|
|
35
|
+
readonly cursor: Cursor.Value;
|
|
36
|
+
readonly lastSuccessAt?: string;
|
|
37
|
+
};
|
|
38
|
+
readonly lastError?: {
|
|
39
|
+
readonly source: "backfill" | "changes";
|
|
40
|
+
readonly operation: "fetch" | "publish" | "checkpoint";
|
|
41
|
+
readonly code: "fetch_failed" | "publish_failed" | "checkpoint_failed";
|
|
42
|
+
readonly message: string;
|
|
43
|
+
readonly at: string;
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
type SyncState = "pending" | "backfilling" | "live" | "error";
|
|
47
|
+
type ResourceSchema = Schema.Decoder<object>;
|
|
48
|
+
/**
|
|
49
|
+
* Takes a resource schema and returns its decoded row object type.
|
|
50
|
+
*
|
|
51
|
+
* @example
|
|
52
|
+
* ```ts
|
|
53
|
+
* const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String });
|
|
54
|
+
* type Product = ResourceRow<typeof ProductSchema>;
|
|
55
|
+
* // Result:
|
|
56
|
+
* // type Product = { readonly id: string; readonly updatedAt: string }
|
|
57
|
+
* ```
|
|
58
|
+
*/
|
|
59
|
+
type ResourceRow<S$1 extends ResourceSchema> = Schema.Schema.Type<S$1>;
|
|
60
|
+
/**
|
|
61
|
+
* Takes a resource schema and returns the string keys available on its row type.
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* ```ts
|
|
65
|
+
* const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String });
|
|
66
|
+
* type ProductField = ResourceField<typeof ProductSchema>;
|
|
67
|
+
* // Result:
|
|
68
|
+
* // type ProductField = "id" | "updatedAt"
|
|
69
|
+
*
|
|
70
|
+
* const key: ResourceField<typeof ProductSchema> = "id";
|
|
71
|
+
* ```
|
|
72
|
+
*/
|
|
73
|
+
type ResourceField<S$1 extends ResourceSchema> = keyof ResourceRow<S$1> & string;
|
|
74
|
+
type FetchPageResult<Row extends object> = {
|
|
75
|
+
readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
|
|
76
|
+
readonly nextPageCursor?: Cursor.Value;
|
|
77
|
+
readonly hasMore: boolean;
|
|
78
|
+
};
|
|
79
|
+
type PageFetch<Row extends object, R = never> = {
|
|
80
|
+
readonly pageCursor: Cursor.Definition;
|
|
81
|
+
readonly cutoff: Cursor.Definition;
|
|
82
|
+
readonly fetch: (input: {
|
|
83
|
+
readonly pageCursor?: Cursor.Value;
|
|
84
|
+
readonly cutoff: Cursor.Value;
|
|
85
|
+
}) => Effect.Effect<FetchPageResult<Row>, ConnectorError, R>;
|
|
86
|
+
};
|
|
87
|
+
type FetchChangesResult<Row extends object> = {
|
|
88
|
+
readonly mutations: ReadonlyArray<ResourceMutation<Row>>;
|
|
89
|
+
readonly cursor: Cursor.Value;
|
|
90
|
+
};
|
|
91
|
+
type ChangesFetch<Row extends object, R = never> = {
|
|
92
|
+
readonly cursor: Cursor.Definition;
|
|
93
|
+
readonly interval?: Duration.Input;
|
|
94
|
+
readonly fetch: (input: {
|
|
95
|
+
readonly cursor: Cursor.Value;
|
|
96
|
+
}) => Effect.Effect<FetchChangesResult<Row>, ConnectorError, R>;
|
|
97
|
+
};
|
|
98
|
+
type WebhookHandler<Row extends object, Payload$1> = {
|
|
99
|
+
readonly schema: Schema.Decoder<Payload$1>;
|
|
100
|
+
handler(input: {
|
|
101
|
+
readonly payload: Payload$1;
|
|
102
|
+
}): Effect.Effect<ReadonlyArray<ResourceMutation<Row>>, ConnectorError>;
|
|
103
|
+
};
|
|
104
|
+
type ResourceDefinition<S$1 extends ResourceSchema = ResourceSchema, Payload$1 = unknown, R = never, Name extends string = string> = {
|
|
105
|
+
readonly name: Name;
|
|
106
|
+
readonly schema: S$1;
|
|
107
|
+
readonly key: ResourceField<NoInfer<S$1>>;
|
|
108
|
+
readonly version: ResourceField<NoInfer<S$1>>;
|
|
109
|
+
readonly partition?: {
|
|
110
|
+
readonly required: boolean;
|
|
111
|
+
};
|
|
112
|
+
/** Read-only validation used before provisioning this resource. */
|
|
113
|
+
readonly check: Effect.Effect<void, ConnectorError>;
|
|
114
|
+
readonly backfill?: PageFetch<ResourceRow<NoInfer<S$1>>, R>;
|
|
115
|
+
readonly changes?: ChangesFetch<ResourceRow<NoInfer<S$1>>, R>;
|
|
116
|
+
readonly webhook?: WebhookHandler<ResourceRow<NoInfer<S$1>>, Payload$1>;
|
|
117
|
+
};
|
|
118
|
+
/**
|
|
119
|
+
* Takes a connector resource tuple and returns a `{ [resourceName]: rowType }`
|
|
120
|
+
* map.
|
|
121
|
+
*
|
|
122
|
+
* @example
|
|
123
|
+
* ```ts
|
|
124
|
+
* type Rows = ResourceRows<[typeof Products, typeof Orders]>;
|
|
125
|
+
* // Result:
|
|
126
|
+
* // type Rows = {
|
|
127
|
+
* // readonly products: Product;
|
|
128
|
+
* // readonly orders: Order;
|
|
129
|
+
* // }
|
|
130
|
+
*
|
|
131
|
+
* type ProductRow = Rows["products"];
|
|
132
|
+
* // Result:
|
|
133
|
+
* // type ProductRow = Product
|
|
134
|
+
* ```
|
|
135
|
+
*/
|
|
136
|
+
type ResourceRows<Resources extends ReadonlyArray<ResourceDefinition>> = { readonly [Resource in Resources[number] as Resource["name"]]: Resource extends ResourceDefinition<infer S, unknown, unknown> ? ResourceRow<S> : never };
|
|
137
|
+
/** Resource names preserved from a connector's resource tuple. */
|
|
138
|
+
type ResourceName<Resources extends ReadonlyArray<ResourceDefinition>> = keyof ResourceRows<Resources> & string;
|
|
139
|
+
/**
|
|
140
|
+
* Takes a resource definition and returns the payload type accepted by
|
|
141
|
+
* `to(resource, payload)`.
|
|
142
|
+
*
|
|
143
|
+
* @example
|
|
144
|
+
* ```ts
|
|
145
|
+
* type ProductPayload = ResourcePayload<typeof Products>;
|
|
146
|
+
* // Result, if Products was defined with a ProductEvent webhook schema:
|
|
147
|
+
* // type ProductPayload = ProductEvent
|
|
148
|
+
* ```
|
|
149
|
+
*/
|
|
150
|
+
type ResourcePayload<R> = R extends ResourceDefinition<ResourceSchema, infer Payload, unknown> ? Payload : never;
|
|
151
|
+
type WebhookAckMode = "after-enqueue" | "after-publish";
|
|
152
|
+
type WebhookRouteContext<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, Payload$1 = unknown> = {
|
|
153
|
+
readonly request: HttpServerRequest.HttpServerRequest;
|
|
154
|
+
readonly rawBody: Uint8Array;
|
|
155
|
+
readonly payload: Payload$1;
|
|
156
|
+
/** Decodes and collects mutations for a resource-owned webhook handler. */
|
|
157
|
+
readonly to: <Resource$1 extends Resources[number]>(resource: Resource$1, payload: ResourcePayload<Resource$1>) => Effect.Effect<void, ConnectorError>;
|
|
158
|
+
};
|
|
159
|
+
type WebhookRoute<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, Payload$1 = unknown> = {
|
|
160
|
+
readonly path: HttpRouter.PathInput;
|
|
161
|
+
readonly ackMode: WebhookAckMode;
|
|
162
|
+
readonly schema: Schema.Decoder<Payload$1>;
|
|
163
|
+
handler(context: WebhookRouteContext<Resources, Payload$1>): Effect.Effect<HttpServerResponse.HttpServerResponse, ConnectorError>;
|
|
164
|
+
};
|
|
165
|
+
type WebhookRouteInput<Resources extends ReadonlyArray<ResourceDefinition>, Payload$1> = WebhookRoute<Resources, Payload$1>;
|
|
166
|
+
type ConnectorDefinition<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>> = {
|
|
167
|
+
readonly name: string;
|
|
168
|
+
readonly title?: string;
|
|
169
|
+
readonly resources: Resources;
|
|
170
|
+
readonly webhooks?: ReadonlyArray<WebhookRoute>;
|
|
171
|
+
};
|
|
172
|
+
//#endregion
|
|
173
|
+
export { WebhookRouteInput as C, WebhookRouteContext as S, ResourceState as _, FetchChangesResult as a, WebhookHandler as b, ResourceBatch as c, ResourceMutation as d, ResourceName as f, ResourceSchema as g, ResourceRows as h, DeleteValue as i, ResourceDefinition as l, ResourceRow as m, ConnectorDefinition as n, FetchPageResult as o, ResourcePayload as p, Cursor as r, PageFetch as s, ChangesFetch as t, ResourceField as u, SyncState as v, WebhookRoute as x, WebhookAckMode as y };
|
|
174
|
+
//# sourceMappingURL=types-B35J5YPu.d.ts.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import "../index-D54ySAKn.js";
|
|
2
|
+
import { C as WebhookRouteInput, S as WebhookRouteContext, x as WebhookRoute, y as WebhookAckMode } from "../types-B35J5YPu.js";
|
|
3
|
+
import "../service-IE5qaf9x.js";
|
|
4
|
+
import "../index-Byr0Lf0C.js";
|
|
5
|
+
import { n as route, r as router } from "../index-fIIBVAai.js";
|
|
6
|
+
export { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput, route, router };
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import "../errors-DHWylqRZ.js";
|
|
2
|
+
import "../metrics-CWPy2Aqc.js";
|
|
3
|
+
import "../constants-BRhIX_rz.js";
|
|
4
|
+
import "../service-DuFiQwid.js";
|
|
5
|
+
import { n as router } from "../server-CXTQ7odl.js";
|
|
6
|
+
import { t as route } from "../webhook-DI-Afxkg.js";
|
|
7
|
+
|
|
8
|
+
export { route, router };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { n as router } from "./server-CXTQ7odl.js";
|
|
3
|
+
|
|
4
|
+
//#region src/webhook/index.ts
|
|
5
|
+
var webhook_exports = /* @__PURE__ */ __export({
|
|
6
|
+
route: () => route,
|
|
7
|
+
router: () => router
|
|
8
|
+
});
|
|
9
|
+
const route = (definition) => definition;
|
|
10
|
+
|
|
11
|
+
//#endregion
|
|
12
|
+
export { webhook_exports as n, route as t };
|
|
13
|
+
//# sourceMappingURL=webhook-DI-Afxkg.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"webhook-DI-Afxkg.js","names":[],"sources":["../src/webhook/index.ts"],"sourcesContent":["import type { ResourceDefinition, WebhookRoute, WebhookRouteInput } from \"../core/types\";\n\nexport { router } from \"./server\";\nexport type { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput } from \"./types\";\n\nexport const route = <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(\n definition: WebhookRouteInput<Resources, Payload>,\n): WebhookRoute<Resources, Payload> => definition;\n"],"mappings":";;;;;;;;AAKA,MAAa,SACX,eACqC"}
|