@useairfoil/connector-kit 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/METRICS.md +78 -0
- package/README.md +450 -88
- package/dist/auth/index.d.ts +2 -0
- package/dist/auth/index.js +3 -0
- package/dist/auth-DvU_CZ7C.js +42 -0
- package/dist/auth-DvU_CZ7C.js.map +1 -0
- package/dist/chunk-Bp6m_JJh.js +13 -0
- package/dist/connector-app/index.d.ts +14 -0
- package/dist/connector-app/index.js +11 -0
- package/dist/connector-app-d7Z-SZLQ.js +64 -0
- package/dist/connector-app-d7Z-SZLQ.js.map +1 -0
- package/dist/constants-BRhIX_rz.js +27 -0
- package/dist/constants-BRhIX_rz.js.map +1 -0
- package/dist/errors/index.d.ts +2 -0
- package/dist/errors/index.js +3 -0
- package/dist/errors-DHWylqRZ.js +8 -0
- package/dist/errors-DHWylqRZ.js.map +1 -0
- package/dist/index-Byr0Lf0C.d.ts +25 -0
- package/dist/index-CBT-JX-T.d.ts +173 -0
- package/dist/index-CTHVudvQ.d.ts +41 -0
- package/dist/index-CTLYQ74P.d.ts +80 -0
- package/dist/index-Coh_SkBs.d.ts +95 -0
- package/dist/index-D54ySAKn.d.ts +14 -0
- package/dist/index-DtJX_tic.d.ts +153 -0
- package/dist/index-J42AZaXU.d.ts +38 -0
- package/dist/index-fIIBVAai.d.ts +25 -0
- package/dist/index-o1-Z3n7V.d.ts +55 -0
- package/dist/index.d.ts +14 -165
- package/dist/index.js +50 -228
- package/dist/index.js.map +1 -1
- package/dist/ingestion/index.d.ts +6 -0
- package/dist/ingestion/index.js +10 -0
- package/dist/ingestion-BQo4cbGw.js +200 -0
- package/dist/ingestion-BQo4cbGw.js.map +1 -0
- package/dist/manifest/index.d.ts +2 -0
- package/dist/manifest/index.js +4 -0
- package/dist/manifest-ByE6YPG0.js +221 -0
- package/dist/manifest-ByE6YPG0.js.map +1 -0
- package/dist/metrics-CVAk1ALR.d.ts +53 -0
- package/dist/metrics-CWPy2Aqc.js +96 -0
- package/dist/metrics-CWPy2Aqc.js.map +1 -0
- package/dist/metrics.d.ts +4 -0
- package/dist/metrics.js +3 -0
- package/dist/publisher/index.d.ts +5 -0
- package/dist/publisher/index.js +6 -0
- package/dist/publisher-BYT3RLVO.js +131 -0
- package/dist/publisher-BYT3RLVO.js.map +1 -0
- package/dist/runtime-config/index.d.ts +2 -0
- package/dist/runtime-config/index.js +4 -0
- package/dist/runtime-config-BpB-d_bF.js +51 -0
- package/dist/runtime-config-BpB-d_bF.js.map +1 -0
- package/dist/server-CXTQ7odl.js +183 -0
- package/dist/server-CXTQ7odl.js.map +1 -0
- package/dist/service-DuFiQwid.js +191 -0
- package/dist/service-DuFiQwid.js.map +1 -0
- package/dist/service-IE5qaf9x.d.ts +32 -0
- package/dist/state-store/index.d.ts +3 -0
- package/dist/state-store/index.js +5 -0
- package/dist/state-store-yl6LiTQ0.js +198 -0
- package/dist/state-store-yl6LiTQ0.js.map +1 -0
- package/dist/status/index.d.ts +5 -0
- package/dist/status/index.js +6 -0
- package/dist/status-B-oZBc33.js +84 -0
- package/dist/status-B-oZBc33.js.map +1 -0
- package/dist/types-B35J5YPu.d.ts +174 -0
- package/dist/webhook/index.d.ts +6 -0
- package/dist/webhook/index.js +8 -0
- package/dist/webhook-DI-Afxkg.js +13 -0
- package/dist/webhook-DI-Afxkg.js.map +1 -0
- package/package.json +66 -8
- package/src/auth/index.ts +1 -0
- package/src/auth/token-cache.ts +54 -0
- package/src/connector-app/check.ts +76 -0
- package/src/connector-app/index.ts +2 -0
- package/src/connector-app/start.ts +42 -0
- package/src/core/builder.ts +81 -9
- package/src/core/index.ts +22 -0
- package/src/core/types.ts +204 -76
- package/src/errors/index.ts +1 -0
- package/src/index.ts +13 -28
- package/src/ingestion/engine.ts +409 -174
- package/src/ingestion/index.ts +1 -0
- package/src/manifest/config.ts +232 -0
- package/src/manifest/index.ts +14 -0
- package/src/manifest/manifest.ts +31 -0
- package/src/manifest/number-schema.ts +17 -0
- package/src/manifest/schema.ts +160 -0
- package/src/metrics.ts +128 -0
- package/src/publisher/console.ts +18 -0
- package/src/publisher/index.ts +7 -0
- package/src/publisher/instrumented.ts +68 -0
- package/src/publisher/service.ts +30 -13
- package/src/publisher/wings.ts +159 -72
- package/src/runtime/context.ts +15 -0
- package/src/runtime-config/constants.ts +27 -0
- package/src/runtime-config/error.ts +13 -0
- package/src/runtime-config/index.ts +3 -0
- package/src/runtime-config/layer.ts +56 -0
- package/src/state-store/index.ts +3 -0
- package/src/state-store/keys.ts +22 -0
- package/src/state-store/layer-key-value-store.ts +132 -0
- package/src/state-store/layer-memory.ts +11 -0
- package/src/state-store/layer-sql.ts +50 -0
- package/src/state-store/schema.ts +57 -0
- package/src/state-store/service.ts +51 -0
- package/src/state-store/state.ts +17 -0
- package/src/status/index.ts +2 -0
- package/src/status/route.ts +77 -0
- package/src/status/types.ts +33 -0
- package/src/telemetry.ts +288 -0
- package/src/webhook/index.ts +8 -0
- package/src/webhook/server.ts +209 -39
- package/src/webhook/types.ts +6 -17
- package/src/ingestion/state-store.ts +0 -36
- package/src/streams/pull-stream.ts +0 -54
- package/src/streams/webhook-queue.ts +0 -14
- /package/src/{core/errors.ts → errors/connector-error.ts} +0 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { r as isPlatformRuntimeKey } from "./constants-BRhIX_rz.js";
|
|
3
|
+
import { Config, Option, Predicate, Redacted, Schema, SchemaTransformation, Struct } from "effect";
|
|
4
|
+
|
|
5
|
+
//#region src/manifest/number-schema.ts
|
|
6
|
+
const makeNumberSchema = (constraints) => {
|
|
7
|
+
if (constraints.integer) return constraints.minimum === void 0 ? Schema.Int : Schema.Int.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
|
|
8
|
+
return constraints.minimum === void 0 ? Schema.Finite : Schema.Finite.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
//#endregion
|
|
12
|
+
//#region src/manifest/config.ts
|
|
13
|
+
const configFieldToConfig = Struct.lambda((field) => field.config);
|
|
14
|
+
const withDefault = (config, defaultValue) => defaultValue === void 0 ? config : Config.withDefault(config, defaultValue);
|
|
15
|
+
/** Defines a required non-empty string, optionally with a default. */
|
|
16
|
+
const string = (options) => ({
|
|
17
|
+
config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
|
|
18
|
+
spec: {
|
|
19
|
+
runtimeKey: options.runtimeKey,
|
|
20
|
+
type: "string",
|
|
21
|
+
required: options.required ?? true,
|
|
22
|
+
secret: false,
|
|
23
|
+
...options.description !== void 0 ? { description: options.description } : {},
|
|
24
|
+
...options.default !== void 0 ? { default: options.default } : {}
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
/** Defines a finite number with optional integer, minimum, and default constraints. */
|
|
28
|
+
const number = (options) => {
|
|
29
|
+
const schema = makeNumberSchema(options);
|
|
30
|
+
const defaultValue = options.default === void 0 ? void 0 : Schema.decodeUnknownSync(schema)(options.default);
|
|
31
|
+
return {
|
|
32
|
+
config: withDefault(Config.schema(schema, options.runtimeKey), defaultValue),
|
|
33
|
+
spec: {
|
|
34
|
+
runtimeKey: options.runtimeKey,
|
|
35
|
+
type: "number",
|
|
36
|
+
required: options.required ?? true,
|
|
37
|
+
secret: false,
|
|
38
|
+
...options.description !== void 0 ? { description: options.description } : {},
|
|
39
|
+
...defaultValue !== void 0 ? { default: defaultValue } : {},
|
|
40
|
+
...options.integer !== void 0 ? { integer: options.integer } : {},
|
|
41
|
+
...options.minimum !== void 0 ? { minimum: options.minimum } : {}
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
};
|
|
45
|
+
/** Defines a boolean, optionally with a default. */
|
|
46
|
+
const boolean = (options) => ({
|
|
47
|
+
config: withDefault(Config.boolean(options.runtimeKey), options.default),
|
|
48
|
+
spec: {
|
|
49
|
+
runtimeKey: options.runtimeKey,
|
|
50
|
+
type: "boolean",
|
|
51
|
+
required: options.required ?? true,
|
|
52
|
+
secret: false,
|
|
53
|
+
...options.description !== void 0 ? { description: options.description } : {},
|
|
54
|
+
...options.default !== void 0 ? { default: options.default } : {}
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
/** Defines a string literal choice whose values are preserved in the inferred type. */
|
|
58
|
+
const select = (options) => ({
|
|
59
|
+
config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
|
|
60
|
+
spec: {
|
|
61
|
+
runtimeKey: options.runtimeKey,
|
|
62
|
+
type: "select",
|
|
63
|
+
required: options.required ?? true,
|
|
64
|
+
secret: false,
|
|
65
|
+
values: options.values,
|
|
66
|
+
...options.description !== void 0 ? { description: options.description } : {},
|
|
67
|
+
...options.default !== void 0 ? { default: options.default } : {}
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
/** Defines a required, non-empty secret decoded as `Redacted<string>`. */
|
|
71
|
+
const secret = (options) => ({
|
|
72
|
+
config: Config.schema(Schema.Redacted(Schema.NonEmptyString), options.runtimeKey),
|
|
73
|
+
spec: {
|
|
74
|
+
runtimeKey: options.runtimeKey,
|
|
75
|
+
type: "string",
|
|
76
|
+
required: options.required ?? true,
|
|
77
|
+
secret: true,
|
|
78
|
+
...options.description !== void 0 ? { description: options.description } : {}
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
/** Makes an existing field optional and represents a missing value as `Option.none()`. */
|
|
82
|
+
const optional = (field) => ({
|
|
83
|
+
config: Config.option(field.config),
|
|
84
|
+
spec: {
|
|
85
|
+
...field.spec,
|
|
86
|
+
required: false
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
/**
|
|
90
|
+
* Defines user-facing connector config once for both Effect runtime loading and
|
|
91
|
+
* frontend form generation. The object keys become logical form/API field names;
|
|
92
|
+
* `runtimeKey` remains the key read by Effect Config.
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```ts
|
|
96
|
+
* const config = define({
|
|
97
|
+
* apiUrl: string({ runtimeKey: "API_URL" }),
|
|
98
|
+
* token: secret({ runtimeKey: "API_TOKEN" }),
|
|
99
|
+
* });
|
|
100
|
+
*
|
|
101
|
+
* const RuntimeConfig = config.config;
|
|
102
|
+
* const formFields = config.spec;
|
|
103
|
+
* ```
|
|
104
|
+
*/
|
|
105
|
+
const define = (fields) => {
|
|
106
|
+
const spec = Object.entries(fields).map(([name, field]) => ({
|
|
107
|
+
name,
|
|
108
|
+
...field.spec
|
|
109
|
+
}));
|
|
110
|
+
const platformField = spec.find((field) => isPlatformRuntimeKey(field.runtimeKey));
|
|
111
|
+
if (platformField !== void 0) throw new Error(`Connector runtime key is platform-owned: ${platformField.runtimeKey}`);
|
|
112
|
+
const configFields = Struct.map(fields, configFieldToConfig);
|
|
113
|
+
return {
|
|
114
|
+
fields: configFields,
|
|
115
|
+
config: Config.unwrap(configFields),
|
|
116
|
+
spec
|
|
117
|
+
};
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
//#endregion
|
|
121
|
+
//#region src/manifest/manifest.ts
|
|
122
|
+
/**
|
|
123
|
+
* Defines browser-safe connector metadata while preserving literal names and capabilities.
|
|
124
|
+
*
|
|
125
|
+
* @example
|
|
126
|
+
* ```ts
|
|
127
|
+
* const manifest = define({
|
|
128
|
+
* name: "producer-example",
|
|
129
|
+
* title: "Example",
|
|
130
|
+
* config: ExampleConfig.spec,
|
|
131
|
+
* resources: [{ name: "orders", capabilities: ["backfill"] }],
|
|
132
|
+
* });
|
|
133
|
+
* ```
|
|
134
|
+
*/
|
|
135
|
+
const define$1 = (manifest) => manifest;
|
|
136
|
+
|
|
137
|
+
//#endregion
|
|
138
|
+
//#region src/manifest/schema.ts
|
|
139
|
+
const numberFromForm = (field) => {
|
|
140
|
+
const numberSchema = makeNumberSchema(field);
|
|
141
|
+
return Schema.Union([numberSchema, Schema.NonEmptyString.pipe(Schema.decodeTo(numberSchema, SchemaTransformation.numberFromString))]);
|
|
142
|
+
};
|
|
143
|
+
const BooleanFromForm = Schema.Union([Schema.Boolean, Schema.Literals(["true", "false"]).pipe(Schema.decodeTo(Schema.Boolean, SchemaTransformation.transform({
|
|
144
|
+
decode: (value) => value === "true",
|
|
145
|
+
encode: (value) => value ? "true" : "false"
|
|
146
|
+
})))]);
|
|
147
|
+
const EmptyStringAsUndefined = Schema.Literal("").pipe(Schema.decodeTo(Schema.Undefined, SchemaTransformation.transform({
|
|
148
|
+
decode: () => void 0,
|
|
149
|
+
encode: () => ""
|
|
150
|
+
})));
|
|
151
|
+
const baseFieldSchema = (field) => {
|
|
152
|
+
switch (field.type) {
|
|
153
|
+
case "number": return numberFromForm(field);
|
|
154
|
+
case "boolean": return BooleanFromForm;
|
|
155
|
+
case "select": return Schema.Literals(field.values);
|
|
156
|
+
case "string": return Schema.NonEmptyString;
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
const fieldSchema = (field) => field.required ? baseFieldSchema(field) : Schema.optional(Schema.Union([EmptyStringAsUndefined, baseFieldSchema(field)]));
|
|
160
|
+
/**
|
|
161
|
+
* Builds an Effect Schema for logical form/API values from a connector manifest.
|
|
162
|
+
*
|
|
163
|
+
* @example
|
|
164
|
+
* ```ts
|
|
165
|
+
* const decode = Schema.decodeUnknownEffect(configSchema(manifest));
|
|
166
|
+
* const values = decode({
|
|
167
|
+
* apiUrl: "https://example.com",
|
|
168
|
+
* retries: "3",
|
|
169
|
+
* });
|
|
170
|
+
* ```
|
|
171
|
+
*/
|
|
172
|
+
const configSchema = (manifest) => {
|
|
173
|
+
const fields = Object.fromEntries(manifest.config.map((field) => [field.name, fieldSchema(field)]));
|
|
174
|
+
return Schema.Struct(fields);
|
|
175
|
+
};
|
|
176
|
+
/** Decodes browser/API config, applies manifest defaults, and rejects undeclared fields. */
|
|
177
|
+
const decodeConfig = (manifest, input) => {
|
|
178
|
+
const defaults = Object.fromEntries(manifest.config.flatMap((field) => field.default === void 0 ? [] : [[field.name, field.default]]));
|
|
179
|
+
const optionalFields = new Set(manifest.config.flatMap((field) => field.required ? [] : [field.name]));
|
|
180
|
+
const normalizedInput = Predicate.isObject(input) ? Object.fromEntries(Object.entries(input).filter(([name, value]) => value !== "" || !optionalFields.has(name))) : input;
|
|
181
|
+
const inputWithDefaults = Predicate.isObject(normalizedInput) ? Object.assign({}, defaults, normalizedInput) : normalizedInput;
|
|
182
|
+
return Schema.decodeUnknownEffect(configSchema(manifest))(inputWithDefaults, { onExcessProperty: "error" });
|
|
183
|
+
};
|
|
184
|
+
/**
|
|
185
|
+
* Maps validated logical values to the flat runtime-key document mounted in a pod.
|
|
186
|
+
* Missing optional values are omitted.
|
|
187
|
+
*
|
|
188
|
+
* @example
|
|
189
|
+
* ```ts
|
|
190
|
+
* toRuntimeDocument(manifest, { apiUrl: "https://example.com" });
|
|
191
|
+
* // { API_URL: "https://example.com" }
|
|
192
|
+
* ```
|
|
193
|
+
*/
|
|
194
|
+
const toRuntimeDocument = (manifest, values) => {
|
|
195
|
+
const document = {};
|
|
196
|
+
for (const field of manifest.config) {
|
|
197
|
+
const value = Reflect.get(values, field.name);
|
|
198
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") document[field.runtimeKey] = value;
|
|
199
|
+
}
|
|
200
|
+
return document;
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
//#endregion
|
|
204
|
+
//#region src/manifest/index.ts
|
|
205
|
+
var manifest_exports = /* @__PURE__ */ __export({
|
|
206
|
+
boolean: () => boolean,
|
|
207
|
+
configSchema: () => configSchema,
|
|
208
|
+
decodeConfig: () => decodeConfig,
|
|
209
|
+
define: () => define$1,
|
|
210
|
+
defineConfig: () => define,
|
|
211
|
+
number: () => number,
|
|
212
|
+
optional: () => optional,
|
|
213
|
+
secret: () => secret,
|
|
214
|
+
select: () => select,
|
|
215
|
+
string: () => string,
|
|
216
|
+
toRuntimeDocument: () => toRuntimeDocument
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
//#endregion
|
|
220
|
+
export { define$1 as a, number as c, select as d, string as f, toRuntimeDocument as i, optional as l, configSchema as n, boolean as o, decodeConfig as r, define as s, manifest_exports as t, secret as u };
|
|
221
|
+
//# sourceMappingURL=manifest-ByE6YPG0.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manifest-ByE6YPG0.js","names":["define","document: Record<string, string | number | boolean>","value: unknown"],"sources":["../src/manifest/number-schema.ts","../src/manifest/config.ts","../src/manifest/manifest.ts","../src/manifest/schema.ts","../src/manifest/index.ts"],"sourcesContent":["import { Schema } from \"effect\";\n\nexport type NumberConstraints = {\n readonly integer?: boolean;\n readonly minimum?: number;\n};\n\nexport const makeNumberSchema = (constraints: NumberConstraints) => {\n if (constraints.integer) {\n return constraints.minimum === undefined\n ? Schema.Int\n : Schema.Int.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));\n }\n return constraints.minimum === undefined\n ? Schema.Finite\n : Schema.Finite.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));\n};\n","import { Config, Option, Redacted, Schema, Struct } from \"effect\";\n\nimport { isPlatformRuntimeKey } from \"../runtime-config/constants\";\nimport { makeNumberSchema, type NumberConstraints } from \"./number-schema\";\n\ntype ConfigFieldSpecBase = {\n readonly runtimeKey: string;\n readonly required: boolean;\n readonly description?: string;\n};\n\ntype ConfigFieldDefinition = ConfigFieldSpecBase &\n (\n | {\n readonly type: \"string\";\n readonly secret: false;\n readonly default?: string;\n readonly values?: never;\n }\n | {\n readonly type: \"string\";\n readonly secret: true;\n readonly default?: never;\n readonly values?: never;\n }\n | {\n readonly type: \"number\";\n readonly secret: false;\n readonly default?: number;\n readonly values?: never;\n readonly integer?: boolean;\n readonly minimum?: number;\n }\n | {\n readonly type: \"boolean\";\n readonly secret: false;\n readonly default?: boolean;\n readonly values?: never;\n }\n | {\n readonly type: \"select\";\n readonly secret: false;\n readonly default?: string;\n readonly values: readonly [string, ...Array<string>];\n }\n );\n\n/** Serializable, browser-safe metadata for one user-provided connector setting. */\nexport type ConfigFieldSpec = ConfigFieldDefinition & { readonly name: string };\n\n/** Pairs an Effect runtime config with the metadata used to generate its form field. */\nexport type ConfigField<A> = {\n readonly config: Config.Config<A>;\n readonly spec: ConfigFieldDefinition;\n};\n\ntype FieldBaseOptions = {\n readonly runtimeKey: string;\n readonly description?: string;\n};\n\ntype FieldOptions<A extends string | number | boolean = string | number | boolean> =\n FieldBaseOptions &\n (\n | {\n readonly required?: true;\n readonly default?: A;\n }\n | {\n readonly required: false;\n readonly default: A;\n }\n );\n\ntype NumberFieldOptions = FieldOptions<number> & NumberConstraints;\n\ntype FieldsRecord = Record<string, ConfigField<unknown>>;\n\n/** Preserves each field's value type while `Struct.map` extracts its Effect Config. */\ninterface ConfigFieldToConfig extends Struct.Lambda {\n <A>(field: ConfigField<A>): Config.Config<A>;\n readonly \"~lambda.out\": this[\"~lambda.in\"] extends ConfigField<infer A>\n ? Config.Config<A>\n : never;\n}\n\nconst configFieldToConfig = Struct.lambda<ConfigFieldToConfig>((field) => field.config);\n\ntype ConfigOf<Fields extends FieldsRecord> = {\n readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A> ? A : never;\n};\n\n/** Effect config and serializable manifest metadata produced from one field definition. */\nexport type ConnectorConfigDef<Fields extends FieldsRecord> = {\n readonly fields: {\n readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A>\n ? Config.Config<A>\n : never;\n };\n readonly config: Config.Config<ConfigOf<Fields>>;\n readonly spec: ReadonlyArray<ConfigFieldSpec>;\n};\n\nexport type ConfigValuesOf<Def extends ConnectorConfigDef<FieldsRecord>> =\n Def extends ConnectorConfigDef<infer Fields> ? ConfigOf<Fields> : never;\n\nconst withDefault = <A extends string | number | boolean>(\n config: Config.Config<A>,\n defaultValue: A | undefined,\n) => (defaultValue === undefined ? config : Config.withDefault(config, defaultValue));\n\n/** Defines a required non-empty string, optionally with a default. */\nexport const string = (options: FieldOptions<string>): ConfigField<string> => ({\n config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"string\",\n required: options.required ?? true,\n secret: false,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(options.default !== undefined ? { default: options.default } : {}),\n },\n});\n\n/** Defines a finite number with optional integer, minimum, and default constraints. */\nexport const number = (options: NumberFieldOptions): ConfigField<number> => {\n const schema = makeNumberSchema(options);\n const defaultValue =\n options.default === undefined ? undefined : Schema.decodeUnknownSync(schema)(options.default);\n\n return {\n config: withDefault(Config.schema(schema, options.runtimeKey), defaultValue),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"number\",\n required: options.required ?? true,\n secret: false,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(defaultValue !== undefined ? { default: defaultValue } : {}),\n ...(options.integer !== undefined ? { integer: options.integer } : {}),\n ...(options.minimum !== undefined ? { minimum: options.minimum } : {}),\n },\n };\n};\n\n/** Defines a boolean, optionally with a default. */\nexport const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> => ({\n config: withDefault(Config.boolean(options.runtimeKey), options.default),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"boolean\",\n required: options.required ?? true,\n secret: false,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(options.default !== undefined ? { default: options.default } : {}),\n },\n});\n\n/** Defines a string literal choice whose values are preserved in the inferred type. */\nexport const select = <const Values extends readonly [string, ...Array<string>]>(\n options: FieldOptions<Values[number]> & { readonly values: Values },\n): ConfigField<Values[number]> => ({\n config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"select\",\n required: options.required ?? true,\n secret: false,\n values: options.values,\n ...(options.description !== undefined ? { description: options.description } : {}),\n ...(options.default !== undefined ? { default: options.default } : {}),\n },\n});\n\n/** Defines a required, non-empty secret decoded as `Redacted<string>`. */\nexport const secret = (\n options: FieldBaseOptions & { readonly required?: true },\n): ConfigField<Redacted.Redacted<string>> => ({\n config: Config.schema(Schema.Redacted(Schema.NonEmptyString), options.runtimeKey),\n spec: {\n runtimeKey: options.runtimeKey,\n type: \"string\",\n required: options.required ?? true,\n secret: true,\n ...(options.description !== undefined ? { description: options.description } : {}),\n },\n});\n\n/** Makes an existing field optional and represents a missing value as `Option.none()`. */\nexport const optional = <A>(field: ConfigField<A>): ConfigField<Option.Option<A>> => ({\n config: Config.option(field.config),\n spec: {\n ...field.spec,\n required: false,\n },\n});\n\n/**\n * Defines user-facing connector config once for both Effect runtime loading and\n * frontend form generation. The object keys become logical form/API field names;\n * `runtimeKey` remains the key read by Effect Config.\n *\n * @example\n * ```ts\n * const config = define({\n * apiUrl: string({ runtimeKey: \"API_URL\" }),\n * token: secret({ runtimeKey: \"API_TOKEN\" }),\n * });\n *\n * const RuntimeConfig = config.config;\n * const formFields = config.spec;\n * ```\n */\nexport const define = <const Fields extends FieldsRecord>(\n fields: Fields,\n): ConnectorConfigDef<Fields> => {\n const spec = Object.entries(fields).map(([name, field]) => ({ name, ...field.spec }));\n const platformField = spec.find((field) => isPlatformRuntimeKey(field.runtimeKey));\n if (platformField !== undefined) {\n throw new Error(`Connector runtime key is platform-owned: ${platformField.runtimeKey}`);\n }\n\n const configFields = Struct.map(fields, configFieldToConfig);\n\n return {\n fields: configFields,\n // Effect's recursive Config.Wrap cannot reduce this generic mapped type,\n // although ConfigFieldToConfig preserves the value type of every property.\n config: Config.unwrap(configFields as Config.Wrap<ConfigOf<Fields>>),\n spec,\n };\n};\n","import type { ConfigFieldSpec } from \"./config\";\n\nexport type ResourceCapability = \"backfill\" | \"changes\" | \"webhook\";\n\n/** Browser-safe connector metadata consumed by the platform and runtime adapters. */\nexport type ConnectorManifest = {\n readonly name: string;\n readonly title: string;\n readonly description?: string;\n readonly config: ReadonlyArray<ConfigFieldSpec>;\n readonly resources: ReadonlyArray<{\n readonly name: string;\n readonly capabilities: ReadonlyArray<ResourceCapability>;\n }>;\n};\n\n/**\n * Defines browser-safe connector metadata while preserving literal names and capabilities.\n *\n * @example\n * ```ts\n * const manifest = define({\n * name: \"producer-example\",\n * title: \"Example\",\n * config: ExampleConfig.spec,\n * resources: [{ name: \"orders\", capabilities: [\"backfill\"] }],\n * });\n * ```\n */\nexport const define = <const Manifest extends ConnectorManifest>(manifest: Manifest): Manifest =>\n manifest;\n","import { Predicate, Schema, SchemaTransformation, type Types } from \"effect\";\n\nimport type { ConfigFieldSpec } from \"./config\";\nimport type { ConnectorManifest } from \"./manifest\";\n\nimport { makeNumberSchema } from \"./number-schema\";\n\ntype FieldValueOf<Field extends ConfigFieldSpec> = Field[\"type\"] extends \"number\"\n ? number\n : Field[\"type\"] extends \"boolean\"\n ? boolean\n : Field[\"values\"] extends ReadonlyArray<string>\n ? Field[\"values\"][number]\n : string;\n\nexport type ConnectorConfigValues<Manifest extends ConnectorManifest> = Types.Simplify<\n {\n readonly [Field in Manifest[\"config\"][number] as Field[\"required\"] extends true\n ? Field[\"name\"]\n : never]: FieldValueOf<Field>;\n } & {\n readonly [Field in Manifest[\"config\"][number] as Field[\"required\"] extends true\n ? never\n : Field[\"name\"]]?: FieldValueOf<Field> | undefined;\n }\n> &\n object;\n\nexport type ConnectorConfigSchema<Manifest extends ConnectorManifest> = Schema.Codec<\n ConnectorConfigValues<Manifest>,\n unknown\n>;\n\n/** Flat primitive document keyed by the runtime keys consumed by Effect Config. */\nexport type RuntimeConfigDocument = Readonly<Record<string, string | number | boolean>>;\n\n// Browser forms submit numbers and booleans as strings, while API callers may\n// already send their canonical JSON representations. These schemas accept both.\nconst numberFromForm = (field: Extract<ConfigFieldSpec, { readonly type: \"number\" }>) => {\n const numberSchema = makeNumberSchema(field);\n return Schema.Union([\n numberSchema,\n Schema.NonEmptyString.pipe(\n Schema.decodeTo(numberSchema, SchemaTransformation.numberFromString),\n ),\n ]);\n};\n\nconst BooleanFromForm = Schema.Union([\n Schema.Boolean,\n Schema.Literals([\"true\", \"false\"]).pipe(\n Schema.decodeTo(\n Schema.Boolean,\n SchemaTransformation.transform({\n decode: (value) => value === \"true\",\n encode: (value) => (value ? \"true\" : \"false\"),\n }),\n ),\n ),\n]);\n\n// An empty optional form control means \"not supplied\", not an empty config value.\nconst EmptyStringAsUndefined = Schema.Literal(\"\").pipe(\n Schema.decodeTo(\n Schema.Undefined,\n SchemaTransformation.transform({ decode: () => undefined, encode: () => \"\" }),\n ),\n);\n\nconst baseFieldSchema = (field: ConfigFieldSpec): Schema.Codec<unknown, unknown> => {\n switch (field.type) {\n case \"number\":\n return numberFromForm(field);\n case \"boolean\":\n return BooleanFromForm;\n case \"select\":\n return Schema.Literals(field.values);\n case \"string\":\n return Schema.NonEmptyString;\n }\n};\n\nconst fieldSchema = (field: ConfigFieldSpec) =>\n field.required\n ? baseFieldSchema(field)\n : Schema.optional(Schema.Union([EmptyStringAsUndefined, baseFieldSchema(field)]));\n\n/**\n * Builds an Effect Schema for logical form/API values from a connector manifest.\n *\n * @example\n * ```ts\n * const decode = Schema.decodeUnknownEffect(configSchema(manifest));\n * const values = decode({\n * apiUrl: \"https://example.com\",\n * retries: \"3\",\n * });\n * ```\n */\nexport const configSchema = <const Manifest extends ConnectorManifest>(\n manifest: Manifest,\n): ConnectorConfigSchema<Manifest> => {\n const fields = Object.fromEntries(\n manifest.config.map((field) => [field.name, fieldSchema(field)]),\n );\n // Object.fromEntries cannot retain keys discovered from the runtime manifest;\n // the public mapped type restores the same relationship for callers.\n return Schema.Struct(fields) as unknown as ConnectorConfigSchema<Manifest>;\n};\n\n/** Decodes browser/API config, applies manifest defaults, and rejects undeclared fields. */\nexport const decodeConfig = <const Manifest extends ConnectorManifest>(\n manifest: Manifest,\n input: unknown,\n) => {\n const defaults = Object.fromEntries(\n manifest.config.flatMap((field) =>\n field.default === undefined ? [] : [[field.name, field.default]],\n ),\n );\n const optionalFields = new Set(\n manifest.config.flatMap((field) => (field.required ? [] : [field.name])),\n );\n const normalizedInput = Predicate.isObject(input)\n ? Object.fromEntries(\n Object.entries(input).filter(([name, value]) => value !== \"\" || !optionalFields.has(name)),\n )\n : input;\n const inputWithDefaults = Predicate.isObject(normalizedInput)\n ? Object.assign({}, defaults, normalizedInput)\n : normalizedInput;\n\n return Schema.decodeUnknownEffect(configSchema(manifest))(inputWithDefaults, {\n onExcessProperty: \"error\",\n });\n};\n\n/**\n * Maps validated logical values to the flat runtime-key document mounted in a pod.\n * Missing optional values are omitted.\n *\n * @example\n * ```ts\n * toRuntimeDocument(manifest, { apiUrl: \"https://example.com\" });\n * // { API_URL: \"https://example.com\" }\n * ```\n */\nexport const toRuntimeDocument = <const Manifest extends ConnectorManifest>(\n manifest: Manifest,\n values: ConnectorConfigValues<Manifest>,\n): RuntimeConfigDocument => {\n const document: Record<string, string | number | boolean> = {};\n for (const field of manifest.config) {\n const value: unknown = Reflect.get(values, field.name);\n if (typeof value === \"string\" || typeof value === \"number\" || typeof value === \"boolean\") {\n document[field.runtimeKey] = value;\n }\n }\n return document;\n};\n","export {\n boolean,\n define as defineConfig,\n number,\n optional,\n secret,\n select,\n string,\n} from \"./config\";\nexport type { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef } from \"./config\";\nexport { define } from \"./manifest\";\nexport type { ConnectorManifest, ResourceCapability } from \"./manifest\";\nexport { configSchema, decodeConfig, toRuntimeDocument } from \"./schema\";\nexport type { ConnectorConfigSchema, ConnectorConfigValues, RuntimeConfigDocument } from \"./schema\";\n"],"mappings":";;;;;AAOA,MAAa,oBAAoB,gBAAmC;AAClE,KAAI,YAAY,QACd,QAAO,YAAY,YAAY,SAC3B,OAAO,MACP,OAAO,IAAI,MAAM,OAAO,uBAAuB,YAAY,QAAQ,CAAC;AAE1E,QAAO,YAAY,YAAY,SAC3B,OAAO,SACP,OAAO,OAAO,MAAM,OAAO,uBAAuB,YAAY,QAAQ,CAAC;;;;;ACuE7E,MAAM,sBAAsB,OAAO,QAA6B,UAAU,MAAM,OAAO;AAoBvF,MAAM,eACJ,QACA,iBACI,iBAAiB,SAAY,SAAS,OAAO,YAAY,QAAQ,aAAa;;AAGpF,MAAa,UAAU,aAAwD;CAC7E,QAAQ,YAAY,OAAO,eAAe,QAAQ,WAAW,EAAE,QAAQ,QAAQ;CAC/E,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACjF,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;EACtE;CACF;;AAGD,MAAa,UAAU,YAAqD;CAC1E,MAAM,SAAS,iBAAiB,QAAQ;CACxC,MAAM,eACJ,QAAQ,YAAY,SAAY,SAAY,OAAO,kBAAkB,OAAO,CAAC,QAAQ,QAAQ;AAE/F,QAAO;EACL,QAAQ,YAAY,OAAO,OAAO,QAAQ,QAAQ,WAAW,EAAE,aAAa;EAC5E,MAAM;GACJ,YAAY,QAAQ;GACpB,MAAM;GACN,UAAU,QAAQ,YAAY;GAC9B,QAAQ;GACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;GACjF,GAAI,iBAAiB,SAAY,EAAE,SAAS,cAAc,GAAG,EAAE;GAC/D,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;GACrE,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;GACtE;EACF;;;AAIH,MAAa,WAAW,aAA0D;CAChF,QAAQ,YAAY,OAAO,QAAQ,QAAQ,WAAW,EAAE,QAAQ,QAAQ;CACxE,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACjF,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;EACtE;CACF;;AAGD,MAAa,UACX,aACiC;CACjC,QAAQ,YAAY,OAAO,SAAS,QAAQ,QAAQ,QAAQ,WAAW,EAAE,QAAQ,QAAQ;CACzF,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,QAAQ,QAAQ;EAChB,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EACjF,GAAI,QAAQ,YAAY,SAAY,EAAE,SAAS,QAAQ,SAAS,GAAG,EAAE;EACtE;CACF;;AAGD,MAAa,UACX,aAC4C;CAC5C,QAAQ,OAAO,OAAO,OAAO,SAAS,OAAO,eAAe,EAAE,QAAQ,WAAW;CACjF,MAAM;EACJ,YAAY,QAAQ;EACpB,MAAM;EACN,UAAU,QAAQ,YAAY;EAC9B,QAAQ;EACR,GAAI,QAAQ,gBAAgB,SAAY,EAAE,aAAa,QAAQ,aAAa,GAAG,EAAE;EAClF;CACF;;AAGD,MAAa,YAAe,WAA0D;CACpF,QAAQ,OAAO,OAAO,MAAM,OAAO;CACnC,MAAM;EACJ,GAAG,MAAM;EACT,UAAU;EACX;CACF;;;;;;;;;;;;;;;;;AAkBD,MAAa,UACX,WAC+B;CAC/B,MAAM,OAAO,OAAO,QAAQ,OAAO,CAAC,KAAK,CAAC,MAAM,YAAY;EAAE;EAAM,GAAG,MAAM;EAAM,EAAE;CACrF,MAAM,gBAAgB,KAAK,MAAM,UAAU,qBAAqB,MAAM,WAAW,CAAC;AAClF,KAAI,kBAAkB,OACpB,OAAM,IAAI,MAAM,4CAA4C,cAAc,aAAa;CAGzF,MAAM,eAAe,OAAO,IAAI,QAAQ,oBAAoB;AAE5D,QAAO;EACL,QAAQ;EAGR,QAAQ,OAAO,OAAO,aAA8C;EACpE;EACD;;;;;;;;;;;;;;;;;;ACzMH,MAAaA,YAAoD,aAC/D;;;;ACQF,MAAM,kBAAkB,UAAiE;CACvF,MAAM,eAAe,iBAAiB,MAAM;AAC5C,QAAO,OAAO,MAAM,CAClB,cACA,OAAO,eAAe,KACpB,OAAO,SAAS,cAAc,qBAAqB,iBAAiB,CACrE,CACF,CAAC;;AAGJ,MAAM,kBAAkB,OAAO,MAAM,CACnC,OAAO,SACP,OAAO,SAAS,CAAC,QAAQ,QAAQ,CAAC,CAAC,KACjC,OAAO,SACL,OAAO,SACP,qBAAqB,UAAU;CAC7B,SAAS,UAAU,UAAU;CAC7B,SAAS,UAAW,QAAQ,SAAS;CACtC,CAAC,CACH,CACF,CACF,CAAC;AAGF,MAAM,yBAAyB,OAAO,QAAQ,GAAG,CAAC,KAChD,OAAO,SACL,OAAO,WACP,qBAAqB,UAAU;CAAE,cAAc;CAAW,cAAc;CAAI,CAAC,CAC9E,CACF;AAED,MAAM,mBAAmB,UAA2D;AAClF,SAAQ,MAAM,MAAd;EACE,KAAK,SACH,QAAO,eAAe,MAAM;EAC9B,KAAK,UACH,QAAO;EACT,KAAK,SACH,QAAO,OAAO,SAAS,MAAM,OAAO;EACtC,KAAK,SACH,QAAO,OAAO;;;AAIpB,MAAM,eAAe,UACnB,MAAM,WACF,gBAAgB,MAAM,GACtB,OAAO,SAAS,OAAO,MAAM,CAAC,wBAAwB,gBAAgB,MAAM,CAAC,CAAC,CAAC;;;;;;;;;;;;;AAcrF,MAAa,gBACX,aACoC;CACpC,MAAM,SAAS,OAAO,YACpB,SAAS,OAAO,KAAK,UAAU,CAAC,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC,CACjE;AAGD,QAAO,OAAO,OAAO,OAAO;;;AAI9B,MAAa,gBACX,UACA,UACG;CACH,MAAM,WAAW,OAAO,YACtB,SAAS,OAAO,SAAS,UACvB,MAAM,YAAY,SAAY,EAAE,GAAG,CAAC,CAAC,MAAM,MAAM,MAAM,QAAQ,CAAC,CACjE,CACF;CACD,MAAM,iBAAiB,IAAI,IACzB,SAAS,OAAO,SAAS,UAAW,MAAM,WAAW,EAAE,GAAG,CAAC,MAAM,KAAK,CAAE,CACzE;CACD,MAAM,kBAAkB,UAAU,SAAS,MAAM,GAC7C,OAAO,YACL,OAAO,QAAQ,MAAM,CAAC,QAAQ,CAAC,MAAM,WAAW,UAAU,MAAM,CAAC,eAAe,IAAI,KAAK,CAAC,CAC3F,GACD;CACJ,MAAM,oBAAoB,UAAU,SAAS,gBAAgB,GACzD,OAAO,OAAO,EAAE,EAAE,UAAU,gBAAgB,GAC5C;AAEJ,QAAO,OAAO,oBAAoB,aAAa,SAAS,CAAC,CAAC,mBAAmB,EAC3E,kBAAkB,SACnB,CAAC;;;;;;;;;;;;AAaJ,MAAa,qBACX,UACA,WAC0B;CAC1B,MAAMC,WAAsD,EAAE;AAC9D,MAAK,MAAM,SAAS,SAAS,QAAQ;EACnC,MAAMC,QAAiB,QAAQ,IAAI,QAAQ,MAAM,KAAK;AACtD,MAAI,OAAO,UAAU,YAAY,OAAO,UAAU,YAAY,OAAO,UAAU,UAC7E,UAAS,MAAM,cAAc;;AAGjC,QAAO"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { v as SyncState } from "./types-B35J5YPu.js";
|
|
2
|
+
import { Effect, Metric } from "effect";
|
|
3
|
+
import * as effect_Duration0 from "effect/Duration";
|
|
4
|
+
|
|
5
|
+
//#region src/metrics.d.ts
|
|
6
|
+
declare namespace metrics_d_exports {
|
|
7
|
+
export { ApiRetryMetricAttrs, ApiRetryReason, BatchOutcome, BatchSource, LastSuccessMetricAttrs, ResourceMetricAttrs, SyncStateAttrs, WebhookMetricAttrs, WebhookOutcome, apiRetriesTotal, batchSize, batchesTotal, entitiesDeletedTotal, entitiesUpsertedTotal, lastSuccessTimestamp, publishDuration, recordApiRetry, recordWebhookRequest, setLastSuccessTimestamp, setSyncState, setWebhookQueueDepth, syncState, webhookQueueDepth, webhookRequestsTotal };
|
|
8
|
+
}
|
|
9
|
+
type BatchSource = "backfill" | "changes" | "webhook";
|
|
10
|
+
type BatchOutcome = "accepted" | "rejected" | "error";
|
|
11
|
+
type ApiRetryReason = "transport" | "timeout" | "rate_limit" | "server_error";
|
|
12
|
+
type WebhookOutcome = "ok" | "read_error" | "invalid_json" | "invalid_payload" | "rejected" | "handler_error";
|
|
13
|
+
type ResourceMetricAttrs = {
|
|
14
|
+
readonly connector: string;
|
|
15
|
+
readonly resource: string;
|
|
16
|
+
readonly source: BatchSource;
|
|
17
|
+
};
|
|
18
|
+
type WebhookMetricAttrs = {
|
|
19
|
+
readonly connector: string;
|
|
20
|
+
readonly path: string;
|
|
21
|
+
readonly outcome: WebhookOutcome;
|
|
22
|
+
};
|
|
23
|
+
type SyncStateAttrs = {
|
|
24
|
+
readonly connector: string;
|
|
25
|
+
readonly resource: string;
|
|
26
|
+
};
|
|
27
|
+
type LastSuccessMetricAttrs = {
|
|
28
|
+
readonly connector: string;
|
|
29
|
+
readonly resource: string;
|
|
30
|
+
readonly source: "backfill" | "changes";
|
|
31
|
+
};
|
|
32
|
+
type ApiRetryMetricAttrs = {
|
|
33
|
+
readonly connector: string;
|
|
34
|
+
readonly reason: ApiRetryReason;
|
|
35
|
+
};
|
|
36
|
+
declare const entitiesUpsertedTotal: Metric.Counter<number>;
|
|
37
|
+
declare const entitiesDeletedTotal: Metric.Counter<number>;
|
|
38
|
+
declare const batchesTotal: Metric.Counter<number>;
|
|
39
|
+
declare const batchSize: Metric.Histogram<number>;
|
|
40
|
+
declare const publishDuration: Metric.Histogram<effect_Duration0.Duration>;
|
|
41
|
+
declare const webhookRequestsTotal: Metric.Counter<number>;
|
|
42
|
+
declare const webhookQueueDepth: Metric.Gauge<number>;
|
|
43
|
+
declare const syncState: Metric.Gauge<number>;
|
|
44
|
+
declare const lastSuccessTimestamp: Metric.Gauge<number>;
|
|
45
|
+
declare const apiRetriesTotal: Metric.Counter<number>;
|
|
46
|
+
declare const recordWebhookRequest: (attrs: WebhookMetricAttrs) => Effect.Effect<void, never, never>;
|
|
47
|
+
declare const setWebhookQueueDepth: (connector: string, depth: number) => Effect.Effect<void, never, never>;
|
|
48
|
+
declare const setLastSuccessTimestamp: (attrs: LastSuccessMetricAttrs, lastSuccessAt?: string) => Effect.Effect<void, never, never>;
|
|
49
|
+
declare const recordApiRetry: (attrs: ApiRetryMetricAttrs) => Effect.Effect<void, never, never>;
|
|
50
|
+
declare const setSyncState: (attrs: SyncStateAttrs, current: SyncState) => Effect.Effect<void, never, never>;
|
|
51
|
+
//#endregion
|
|
52
|
+
export { syncState as C, setWebhookQueueDepth as S, webhookRequestsTotal as T, publishDuration as _, LastSuccessMetricAttrs as a, setLastSuccessTimestamp as b, WebhookMetricAttrs as c, batchSize as d, batchesTotal as f, metrics_d_exports as g, lastSuccessTimestamp as h, BatchSource as i, WebhookOutcome as l, entitiesUpsertedTotal as m, ApiRetryReason as n, ResourceMetricAttrs as o, entitiesDeletedTotal as p, BatchOutcome as r, SyncStateAttrs as s, ApiRetryMetricAttrs as t, apiRetriesTotal as u, recordApiRetry as v, webhookQueueDepth as w, setSyncState as x, recordWebhookRequest as y };
|
|
53
|
+
//# sourceMappingURL=metrics-CVAk1ALR.d.ts.map
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { DateTime, Effect, Metric, Option } from "effect";
|
|
3
|
+
|
|
4
|
+
//#region src/metrics.ts
|
|
5
|
+
var metrics_exports = /* @__PURE__ */ __export({
|
|
6
|
+
apiRetriesTotal: () => apiRetriesTotal,
|
|
7
|
+
batchSize: () => batchSize,
|
|
8
|
+
batchesTotal: () => batchesTotal,
|
|
9
|
+
entitiesDeletedTotal: () => entitiesDeletedTotal,
|
|
10
|
+
entitiesUpsertedTotal: () => entitiesUpsertedTotal,
|
|
11
|
+
lastSuccessTimestamp: () => lastSuccessTimestamp,
|
|
12
|
+
publishDuration: () => publishDuration,
|
|
13
|
+
recordApiRetry: () => recordApiRetry,
|
|
14
|
+
recordWebhookRequest: () => recordWebhookRequest,
|
|
15
|
+
setLastSuccessTimestamp: () => setLastSuccessTimestamp,
|
|
16
|
+
setSyncState: () => setSyncState,
|
|
17
|
+
setWebhookQueueDepth: () => setWebhookQueueDepth,
|
|
18
|
+
syncState: () => syncState,
|
|
19
|
+
webhookQueueDepth: () => webhookQueueDepth,
|
|
20
|
+
webhookRequestsTotal: () => webhookRequestsTotal
|
|
21
|
+
});
|
|
22
|
+
const entitiesUpsertedTotal = Metric.counter("airfoil_connector_entities_upserted_total", {
|
|
23
|
+
description: "Total entity upsert mutations published by connector sources",
|
|
24
|
+
incremental: true
|
|
25
|
+
});
|
|
26
|
+
const entitiesDeletedTotal = Metric.counter("airfoil_connector_entities_deleted_total", {
|
|
27
|
+
description: "Total entity delete mutations published by connector sources",
|
|
28
|
+
incremental: true
|
|
29
|
+
});
|
|
30
|
+
const batchesTotal = Metric.counter("airfoil_connector_batches_total", {
|
|
31
|
+
description: "Total resource batches published by connector sources",
|
|
32
|
+
incremental: true
|
|
33
|
+
});
|
|
34
|
+
const batchSize = Metric.histogram("airfoil_connector_batch_size", {
|
|
35
|
+
description: "Distribution of resource mutation batch sizes",
|
|
36
|
+
boundaries: [
|
|
37
|
+
1,
|
|
38
|
+
5,
|
|
39
|
+
10,
|
|
40
|
+
25,
|
|
41
|
+
50,
|
|
42
|
+
100,
|
|
43
|
+
250,
|
|
44
|
+
500,
|
|
45
|
+
1e3
|
|
46
|
+
]
|
|
47
|
+
});
|
|
48
|
+
const publishDuration = Metric.timer("airfoil_connector_publish_duration_ms", {
|
|
49
|
+
description: "Publisher call duration in milliseconds",
|
|
50
|
+
boundaries: [
|
|
51
|
+
1,
|
|
52
|
+
5,
|
|
53
|
+
10,
|
|
54
|
+
25,
|
|
55
|
+
50,
|
|
56
|
+
100,
|
|
57
|
+
250,
|
|
58
|
+
500,
|
|
59
|
+
1e3,
|
|
60
|
+
2500,
|
|
61
|
+
5e3,
|
|
62
|
+
1e4
|
|
63
|
+
]
|
|
64
|
+
});
|
|
65
|
+
const webhookRequestsTotal = Metric.counter("airfoil_connector_webhook_requests_total", {
|
|
66
|
+
description: "Total webhook HTTP requests handled by connector routes",
|
|
67
|
+
incremental: true
|
|
68
|
+
});
|
|
69
|
+
const webhookQueueDepth = Metric.gauge("airfoil_connector_webhook_queue_depth", { description: "Current queued webhook batches waiting for publish" });
|
|
70
|
+
const syncState = Metric.gauge("airfoil_connector_sync_state", { description: "Connector resource sync state as a Prometheus state-set gauge" });
|
|
71
|
+
const lastSuccessTimestamp = Metric.gauge("airfoil_connector_last_success_timestamp_seconds", { description: "Unix timestamp of the last successful durable source checkpoint" });
|
|
72
|
+
const apiRetriesTotal = Metric.counter("airfoil_connector_api_retries_total", {
|
|
73
|
+
description: "Total provider API retries performed by the connector",
|
|
74
|
+
incremental: true
|
|
75
|
+
});
|
|
76
|
+
const recordWebhookRequest = (attrs) => Metric.update(Metric.withAttributes(webhookRequestsTotal, attrs), 1);
|
|
77
|
+
const setWebhookQueueDepth = (connector, depth) => Metric.update(Metric.withAttributes(webhookQueueDepth, { connector }), depth);
|
|
78
|
+
const setLastSuccessTimestamp = (attrs, lastSuccessAt) => {
|
|
79
|
+
const timestamp = lastSuccessAt === void 0 ? 0 : DateTime.make(lastSuccessAt).pipe(Option.map((dateTime) => DateTime.toEpochMillis(dateTime) / 1e3), Option.getOrElse(() => 0));
|
|
80
|
+
return Metric.update(Metric.withAttributes(lastSuccessTimestamp, attrs), timestamp);
|
|
81
|
+
};
|
|
82
|
+
const recordApiRetry = (attrs) => Metric.update(Metric.withAttributes(apiRetriesTotal, attrs), 1);
|
|
83
|
+
const syncStates = [
|
|
84
|
+
"pending",
|
|
85
|
+
"backfilling",
|
|
86
|
+
"live",
|
|
87
|
+
"error"
|
|
88
|
+
];
|
|
89
|
+
const setSyncState = (attrs, current) => Effect.forEach(syncStates, (state) => Metric.update(Metric.withAttributes(syncState, {
|
|
90
|
+
...attrs,
|
|
91
|
+
state
|
|
92
|
+
}), state === current ? 1 : 0), { discard: true });
|
|
93
|
+
|
|
94
|
+
//#endregion
|
|
95
|
+
export { entitiesUpsertedTotal as a, publishDuration as c, setLastSuccessTimestamp as d, setSyncState as f, webhookRequestsTotal as g, webhookQueueDepth as h, entitiesDeletedTotal as i, recordApiRetry as l, syncState as m, batchSize as n, lastSuccessTimestamp as o, setWebhookQueueDepth as p, batchesTotal as r, metrics_exports as s, apiRetriesTotal as t, recordWebhookRequest as u };
|
|
96
|
+
//# sourceMappingURL=metrics-CWPy2Aqc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics-CWPy2Aqc.js","names":[],"sources":["../src/metrics.ts"],"sourcesContent":["import { DateTime, Effect, Metric, Option } from \"effect\";\n\nimport type { SyncState } from \"./core/types\";\n\nexport type BatchSource = \"backfill\" | \"changes\" | \"webhook\";\nexport type BatchOutcome = \"accepted\" | \"rejected\" | \"error\";\nexport type ApiRetryReason = \"transport\" | \"timeout\" | \"rate_limit\" | \"server_error\";\nexport type WebhookOutcome =\n | \"ok\"\n | \"read_error\"\n | \"invalid_json\"\n | \"invalid_payload\"\n | \"rejected\"\n | \"handler_error\";\n\nexport type ResourceMetricAttrs = {\n readonly connector: string;\n readonly resource: string;\n readonly source: BatchSource;\n};\n\nexport type WebhookMetricAttrs = {\n readonly connector: string;\n readonly path: string;\n readonly outcome: WebhookOutcome;\n};\n\nexport type SyncStateAttrs = {\n readonly connector: string;\n readonly resource: string;\n};\n\nexport type LastSuccessMetricAttrs = {\n readonly connector: string;\n readonly resource: string;\n readonly source: \"backfill\" | \"changes\";\n};\n\nexport type ApiRetryMetricAttrs = {\n readonly connector: string;\n readonly reason: ApiRetryReason;\n};\n\nexport const entitiesUpsertedTotal = Metric.counter(\"airfoil_connector_entities_upserted_total\", {\n description: \"Total entity upsert mutations published by connector sources\",\n incremental: true,\n});\n\nexport const entitiesDeletedTotal = Metric.counter(\"airfoil_connector_entities_deleted_total\", {\n description: \"Total entity delete mutations published by connector sources\",\n incremental: true,\n});\n\nexport const batchesTotal = Metric.counter(\"airfoil_connector_batches_total\", {\n description: \"Total resource batches published by connector sources\",\n incremental: true,\n});\n\nexport const batchSize = Metric.histogram(\"airfoil_connector_batch_size\", {\n description: \"Distribution of resource mutation batch sizes\",\n boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],\n});\n\nexport const publishDuration = Metric.timer(\"airfoil_connector_publish_duration_ms\", {\n description: \"Publisher call duration in milliseconds\",\n boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],\n});\n\nexport const webhookRequestsTotal = Metric.counter(\"airfoil_connector_webhook_requests_total\", {\n description: \"Total webhook HTTP requests handled by connector routes\",\n incremental: true,\n});\n\nexport const webhookQueueDepth = Metric.gauge(\"airfoil_connector_webhook_queue_depth\", {\n description: \"Current queued webhook batches waiting for publish\",\n});\n\nexport const syncState = Metric.gauge(\"airfoil_connector_sync_state\", {\n description: \"Connector resource sync state as a Prometheus state-set gauge\",\n});\n\nexport const lastSuccessTimestamp = Metric.gauge(\n \"airfoil_connector_last_success_timestamp_seconds\",\n {\n description: \"Unix timestamp of the last successful durable source checkpoint\",\n },\n);\n\nexport const apiRetriesTotal = Metric.counter(\"airfoil_connector_api_retries_total\", {\n description: \"Total provider API retries performed by the connector\",\n incremental: true,\n});\n\nexport const recordWebhookRequest = (attrs: WebhookMetricAttrs) =>\n Metric.update(Metric.withAttributes(webhookRequestsTotal, attrs), 1);\n\nexport const setWebhookQueueDepth = (connector: string, depth: number) =>\n Metric.update(Metric.withAttributes(webhookQueueDepth, { connector }), depth);\n\nexport const setLastSuccessTimestamp = (attrs: LastSuccessMetricAttrs, lastSuccessAt?: string) => {\n const timestamp =\n lastSuccessAt === undefined\n ? 0\n : DateTime.make(lastSuccessAt).pipe(\n Option.map((dateTime) => DateTime.toEpochMillis(dateTime) / 1000),\n Option.getOrElse(() => 0),\n );\n return Metric.update(Metric.withAttributes(lastSuccessTimestamp, attrs), timestamp);\n};\n\nexport const recordApiRetry = (attrs: ApiRetryMetricAttrs) =>\n Metric.update(Metric.withAttributes(apiRetriesTotal, attrs), 1);\n\nconst syncStates = [\"pending\", \"backfilling\", \"live\", \"error\"] as const;\n\nexport const setSyncState = (attrs: SyncStateAttrs, current: SyncState) =>\n Effect.forEach(\n syncStates,\n (state) =>\n Metric.update(\n Metric.withAttributes(syncState, {\n ...attrs,\n state,\n }),\n state === current ? 1 : 0,\n ),\n { discard: true },\n );\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA2CA,MAAa,wBAAwB,OAAO,QAAQ,6CAA6C;CAC/F,aAAa;CACb,aAAa;CACd,CAAC;AAEF,MAAa,uBAAuB,OAAO,QAAQ,4CAA4C;CAC7F,aAAa;CACb,aAAa;CACd,CAAC;AAEF,MAAa,eAAe,OAAO,QAAQ,mCAAmC;CAC5E,aAAa;CACb,aAAa;CACd,CAAC;AAEF,MAAa,YAAY,OAAO,UAAU,gCAAgC;CACxE,aAAa;CACb,YAAY;EAAC;EAAG;EAAG;EAAI;EAAI;EAAI;EAAK;EAAK;EAAK;EAAK;CACpD,CAAC;AAEF,MAAa,kBAAkB,OAAO,MAAM,yCAAyC;CACnF,aAAa;CACb,YAAY;EAAC;EAAG;EAAG;EAAI;EAAI;EAAI;EAAK;EAAK;EAAK;EAAM;EAAM;EAAM;EAAM;CACvE,CAAC;AAEF,MAAa,uBAAuB,OAAO,QAAQ,4CAA4C;CAC7F,aAAa;CACb,aAAa;CACd,CAAC;AAEF,MAAa,oBAAoB,OAAO,MAAM,yCAAyC,EACrF,aAAa,sDACd,CAAC;AAEF,MAAa,YAAY,OAAO,MAAM,gCAAgC,EACpE,aAAa,iEACd,CAAC;AAEF,MAAa,uBAAuB,OAAO,MACzC,oDACA,EACE,aAAa,mEACd,CACF;AAED,MAAa,kBAAkB,OAAO,QAAQ,uCAAuC;CACnF,aAAa;CACb,aAAa;CACd,CAAC;AAEF,MAAa,wBAAwB,UACnC,OAAO,OAAO,OAAO,eAAe,sBAAsB,MAAM,EAAE,EAAE;AAEtE,MAAa,wBAAwB,WAAmB,UACtD,OAAO,OAAO,OAAO,eAAe,mBAAmB,EAAE,WAAW,CAAC,EAAE,MAAM;AAE/E,MAAa,2BAA2B,OAA+B,kBAA2B;CAChG,MAAM,YACJ,kBAAkB,SACd,IACA,SAAS,KAAK,cAAc,CAAC,KAC3B,OAAO,KAAK,aAAa,SAAS,cAAc,SAAS,GAAG,IAAK,EACjE,OAAO,gBAAgB,EAAE,CAC1B;AACP,QAAO,OAAO,OAAO,OAAO,eAAe,sBAAsB,MAAM,EAAE,UAAU;;AAGrF,MAAa,kBAAkB,UAC7B,OAAO,OAAO,OAAO,eAAe,iBAAiB,MAAM,EAAE,EAAE;AAEjE,MAAM,aAAa;CAAC;CAAW;CAAe;CAAQ;CAAQ;AAE9D,MAAa,gBAAgB,OAAuB,YAClD,OAAO,QACL,aACC,UACC,OAAO,OACL,OAAO,eAAe,WAAW;CAC/B,GAAG;CACH;CACD,CAAC,EACF,UAAU,UAAU,IAAI,EACzB,EACH,EAAE,SAAS,MAAM,CAClB"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import "./index-D54ySAKn.js";
|
|
2
|
+
import "./types-B35J5YPu.js";
|
|
3
|
+
import { C as syncState, S as setWebhookQueueDepth, T as webhookRequestsTotal, _ as publishDuration, a as LastSuccessMetricAttrs, b as setLastSuccessTimestamp, c as WebhookMetricAttrs, d as batchSize, f as batchesTotal, h as lastSuccessTimestamp, i as BatchSource, l as WebhookOutcome, m as entitiesUpsertedTotal, n as ApiRetryReason, o as ResourceMetricAttrs, p as entitiesDeletedTotal, r as BatchOutcome, s as SyncStateAttrs, t as ApiRetryMetricAttrs, u as apiRetriesTotal, v as recordApiRetry, w as webhookQueueDepth, x as setSyncState, y as recordWebhookRequest } from "./metrics-CVAk1ALR.js";
|
|
4
|
+
export { ApiRetryMetricAttrs, ApiRetryReason, BatchOutcome, BatchSource, LastSuccessMetricAttrs, ResourceMetricAttrs, SyncStateAttrs, WebhookMetricAttrs, WebhookOutcome, apiRetriesTotal, batchSize, batchesTotal, entitiesDeletedTotal, entitiesUpsertedTotal, lastSuccessTimestamp, publishDuration, recordApiRetry, recordWebhookRequest, setLastSuccessTimestamp, setSyncState, setWebhookQueueDepth, syncState, webhookQueueDepth, webhookRequestsTotal };
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { a as entitiesUpsertedTotal, c as publishDuration, d as setLastSuccessTimestamp, f as setSyncState, g as webhookRequestsTotal, h as webhookQueueDepth, i as entitiesDeletedTotal, l as recordApiRetry, m as syncState, n as batchSize, o as lastSuccessTimestamp, p as setWebhookQueueDepth, r as batchesTotal, t as apiRetriesTotal, u as recordWebhookRequest } from "./metrics-CWPy2Aqc.js";
|
|
2
|
+
|
|
3
|
+
export { apiRetriesTotal, batchSize, batchesTotal, entitiesDeletedTotal, entitiesUpsertedTotal, lastSuccessTimestamp, publishDuration, recordApiRetry, recordWebhookRequest, setLastSuccessTimestamp, setSyncState, setWebhookQueueDepth, syncState, webhookQueueDepth, webhookRequestsTotal };
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import "../index-D54ySAKn.js";
|
|
2
|
+
import { c as ResourceBatch } from "../types-B35J5YPu.js";
|
|
3
|
+
import { a as PublisherService, i as Publisher, n as PublishOptions, r as PublishSource, t as PublishAck } from "../service-IE5qaf9x.js";
|
|
4
|
+
import { i as layerWings, n as layerConsole, r as WingsPublisherConfig } from "../index-Byr0Lf0C.js";
|
|
5
|
+
export { PublishAck, PublishOptions, PublishSource, Publisher, PublisherService, ResourceBatch, WingsPublisherConfig, layerConsole, layerWings };
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { t as __export } from "./chunk-Bp6m_JJh.js";
|
|
2
|
+
import { t as ConnectorError } from "./errors-DHWylqRZ.js";
|
|
3
|
+
import { a as SpanName, t as Publisher } from "./service-DuFiQwid.js";
|
|
4
|
+
import { Effect, Layer } from "effect";
|
|
5
|
+
import * as Wings from "@useairfoil/wings";
|
|
6
|
+
|
|
7
|
+
//#region src/publisher/console.ts
|
|
8
|
+
const layerConsole = Layer.succeed(Publisher)({ publish: ({ resource, source, batch }) => Effect.gen(function* () {
|
|
9
|
+
yield* Effect.logInfo(`[publisher] -> Source: ${source} | Resource: ${resource}`).pipe(Effect.annotateLogs({
|
|
10
|
+
mutations: batch.mutations.length,
|
|
11
|
+
cursor: batch.cursor,
|
|
12
|
+
source
|
|
13
|
+
}));
|
|
14
|
+
return {
|
|
15
|
+
status: "accepted",
|
|
16
|
+
resource
|
|
17
|
+
};
|
|
18
|
+
}) });
|
|
19
|
+
|
|
20
|
+
//#endregion
|
|
21
|
+
//#region src/publisher/wings.ts
|
|
22
|
+
const normalizeMapping = (mapping) => typeof mapping === "string" ? { name: mapping } : mapping;
|
|
23
|
+
const buildRecordBatch = (rows) => {
|
|
24
|
+
const [batch] = Wings.Arrow.tableFromJSON(Array.from(rows)).batches;
|
|
25
|
+
return batch ? Effect.succeed(batch) : Effect.fail(new ConnectorError({ message: "No rows to publish" }));
|
|
26
|
+
};
|
|
27
|
+
const validateTableMapping = (options) => Effect.try({
|
|
28
|
+
try: () => {
|
|
29
|
+
const keyField = Wings.TableUtils.getKeyField(options.table);
|
|
30
|
+
const versionField = Wings.TableUtils.getVersionField(options.table);
|
|
31
|
+
const partitionField = Wings.TableUtils.getPartitionField(options.table);
|
|
32
|
+
if (keyField.name !== options.resource.key) throw new Error(`Resource ${options.resource.name} key '${options.resource.key}' does not match Wings table key '${keyField.name}'`);
|
|
33
|
+
if (versionField.name !== options.resource.version) throw new Error(`Resource ${options.resource.name} version '${options.resource.version}' does not match Wings table version '${versionField.name}'`);
|
|
34
|
+
if (options.resource.partition?.required === true && !options.partitionValue) throw new Error(`Resource ${options.resource.name} requires a partition value`);
|
|
35
|
+
if (partitionField && !options.partitionValue) throw new Error(`Wings table ${options.table.name} is partitioned but no partition value was provided`);
|
|
36
|
+
if (!partitionField && options.partitionValue) throw new Error(`Wings table ${options.table.name} is not partitioned but a partition value was provided`);
|
|
37
|
+
return {
|
|
38
|
+
keyField,
|
|
39
|
+
versionField
|
|
40
|
+
};
|
|
41
|
+
},
|
|
42
|
+
catch: (cause) => new ConnectorError({
|
|
43
|
+
message: `Invalid Wings table mapping for ${options.resource.name}`,
|
|
44
|
+
cause
|
|
45
|
+
})
|
|
46
|
+
});
|
|
47
|
+
const accepted = (entry) => ({
|
|
48
|
+
status: "accepted",
|
|
49
|
+
resource: entry.resource.name,
|
|
50
|
+
partition: entry.partitionValue
|
|
51
|
+
});
|
|
52
|
+
const rejected = (entry, reason, rejectedRows) => ({
|
|
53
|
+
status: "rejected",
|
|
54
|
+
resource: entry.resource.name,
|
|
55
|
+
reason,
|
|
56
|
+
rejectedRows,
|
|
57
|
+
partition: entry.partitionValue
|
|
58
|
+
});
|
|
59
|
+
const layerWings = (config) => Layer.effect(Publisher)(Effect.gen(function* () {
|
|
60
|
+
const entries = /* @__PURE__ */ new Map();
|
|
61
|
+
const clusterClient = yield* Wings.WingsClient.clusterClient;
|
|
62
|
+
for (const resource of config.connector.resources) {
|
|
63
|
+
const mappingInput = config.tables[resource.name];
|
|
64
|
+
if (!mappingInput) return yield* Effect.fail(new ConnectorError({ message: `Missing table for ${resource.name}` }));
|
|
65
|
+
const mapping = normalizeMapping(mappingInput);
|
|
66
|
+
const table = yield* clusterClient.getTable({ name: mapping.name }).pipe(Effect.mapError((error) => new ConnectorError({
|
|
67
|
+
message: `Failed to resolve Wings table for ${resource.name}`,
|
|
68
|
+
cause: error
|
|
69
|
+
})));
|
|
70
|
+
const { keyField, versionField } = yield* validateTableMapping({
|
|
71
|
+
resource,
|
|
72
|
+
table,
|
|
73
|
+
partitionValue: mapping.partitionValue
|
|
74
|
+
});
|
|
75
|
+
const publisher = yield* Wings.WingsClient.publisher({
|
|
76
|
+
table,
|
|
77
|
+
partitionValue: mapping.partitionValue
|
|
78
|
+
}).pipe(Effect.mapError((error) => new ConnectorError({
|
|
79
|
+
message: error.message,
|
|
80
|
+
cause: error
|
|
81
|
+
})));
|
|
82
|
+
entries.set(resource.name, {
|
|
83
|
+
resource,
|
|
84
|
+
table,
|
|
85
|
+
publisher,
|
|
86
|
+
partitionValue: mapping.partitionValue,
|
|
87
|
+
keyField,
|
|
88
|
+
versionField
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
const pushRows = (entry, operation, rows) => Effect.gen(function* () {
|
|
92
|
+
if (rows.length === 0) return accepted(entry);
|
|
93
|
+
const batch = yield* buildRecordBatch(rows);
|
|
94
|
+
return yield* entry.publisher.push({
|
|
95
|
+
operation,
|
|
96
|
+
batch,
|
|
97
|
+
partitionValue: entry.partitionValue
|
|
98
|
+
}).pipe(Effect.matchEffect({
|
|
99
|
+
onFailure: (error) => Effect.succeed(rejected(entry, error.message, rows.length)),
|
|
100
|
+
onSuccess: () => Effect.succeed(accepted(entry))
|
|
101
|
+
}));
|
|
102
|
+
});
|
|
103
|
+
const service = { publish: Effect.fn(SpanName.publish, { kind: "producer" })(function* ({ resource, source: _source, batch }) {
|
|
104
|
+
const entry = entries.get(resource);
|
|
105
|
+
if (!entry) return yield* Effect.fail(new ConnectorError({ message: `Unknown resource ${resource}` }));
|
|
106
|
+
if (batch.mutations.length === 0) return accepted(entry);
|
|
107
|
+
const upserts = [];
|
|
108
|
+
const deletes = [];
|
|
109
|
+
for (const mutation of batch.mutations) if (mutation.op === "upsert") upserts.push(mutation.row);
|
|
110
|
+
else deletes.push({
|
|
111
|
+
[entry.keyField.name]: mutation.key,
|
|
112
|
+
[entry.versionField.name]: mutation.version
|
|
113
|
+
});
|
|
114
|
+
const upsertAck = yield* pushRows(entry, "upsert", upserts);
|
|
115
|
+
if (upsertAck.status === "rejected") return upsertAck;
|
|
116
|
+
return yield* pushRows(entry, "delete", deletes);
|
|
117
|
+
}) };
|
|
118
|
+
return Publisher.of(service);
|
|
119
|
+
}));
|
|
120
|
+
|
|
121
|
+
//#endregion
|
|
122
|
+
//#region src/publisher/index.ts
|
|
123
|
+
var publisher_exports = /* @__PURE__ */ __export({
|
|
124
|
+
Publisher: () => Publisher,
|
|
125
|
+
layerConsole: () => layerConsole,
|
|
126
|
+
layerWings: () => layerWings
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
//#endregion
|
|
130
|
+
export { layerWings as n, layerConsole as r, publisher_exports as t };
|
|
131
|
+
//# sourceMappingURL=publisher-BYT3RLVO.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"publisher-BYT3RLVO.js","names":["service: PublisherService","upserts: Rows[]","deletes: Rows[]"],"sources":["../src/publisher/console.ts","../src/publisher/wings.ts","../src/publisher/index.ts"],"sourcesContent":["import { Effect, Layer } from \"effect\";\n\nimport { Publisher } from \"./service\";\n\nexport const layerConsole = Layer.succeed(Publisher)({\n publish: ({ resource, source, batch }) =>\n Effect.gen(function* () {\n yield* Effect.logInfo(`[publisher] -> Source: ${source} | Resource: ${resource}`).pipe(\n Effect.annotateLogs({\n mutations: batch.mutations.length,\n cursor: batch.cursor,\n source,\n }),\n );\n\n return { status: \"accepted\" as const, resource };\n }),\n});\n","import * as Wings from \"@useairfoil/wings\";\nimport { Effect, Layer } from \"effect\";\n\nimport type { ConnectorDefinition, ResourceDefinition } from \"../core/types\";\n\nimport { ConnectorError } from \"../errors\";\nimport { SpanName } from \"../telemetry\";\nimport { Publisher, type PublishAck, type PublisherService } from \"./service\";\n\ntype Rows = Record<string, unknown>;\n\nexport type WingsTableMapping = {\n readonly name: string;\n readonly partitionValue?: Wings.PartitionValue.PartitionValue;\n};\n\nexport type WingsPublisherConfig = {\n readonly connector: ConnectorDefinition;\n readonly tables: Record<string, string | WingsTableMapping>;\n};\n\ntype PublisherEntry = {\n readonly resource: ResourceDefinition;\n readonly table: Wings.Cluster.Table.Table;\n readonly publisher: Wings.WingsClient.Publisher;\n readonly partitionValue?: Wings.PartitionValue.PartitionValue;\n readonly keyField: Wings.Cluster.ArrowType.Field;\n readonly versionField: Wings.Cluster.ArrowType.Field;\n};\n\nconst normalizeMapping = (mapping: string | WingsTableMapping): WingsTableMapping =>\n typeof mapping === \"string\" ? { name: mapping } : mapping;\n\nconst buildRecordBatch = (rows: ReadonlyArray<Rows>) => {\n const table = Wings.Arrow.tableFromJSON(Array.from(rows));\n const [batch] = table.batches;\n return batch\n ? Effect.succeed(batch)\n : Effect.fail(new ConnectorError({ message: \"No rows to publish\" }));\n};\n\nconst validateTableMapping = (options: {\n readonly resource: ResourceDefinition;\n readonly table: Wings.Cluster.Table.Table;\n readonly partitionValue?: Wings.PartitionValue.PartitionValue;\n}) =>\n Effect.try({\n try: () => {\n const keyField = Wings.TableUtils.getKeyField(options.table);\n const versionField = Wings.TableUtils.getVersionField(options.table);\n const partitionField = Wings.TableUtils.getPartitionField(options.table);\n\n if (keyField.name !== options.resource.key) {\n throw new Error(\n `Resource ${options.resource.name} key '${options.resource.key}' does not match Wings table key '${keyField.name}'`,\n );\n }\n if (versionField.name !== options.resource.version) {\n throw new Error(\n `Resource ${options.resource.name} version '${options.resource.version}' does not match Wings table version '${versionField.name}'`,\n );\n }\n if (options.resource.partition?.required === true && !options.partitionValue) {\n throw new Error(`Resource ${options.resource.name} requires a partition value`);\n }\n if (partitionField && !options.partitionValue) {\n throw new Error(\n `Wings table ${options.table.name} is partitioned but no partition value was provided`,\n );\n }\n if (!partitionField && options.partitionValue) {\n throw new Error(\n `Wings table ${options.table.name} is not partitioned but a partition value was provided`,\n );\n }\n\n return { keyField, versionField };\n },\n catch: (cause) =>\n new ConnectorError({\n message: `Invalid Wings table mapping for ${options.resource.name}`,\n cause,\n }),\n });\n\nconst accepted = (entry: PublisherEntry): PublishAck => ({\n status: \"accepted\",\n resource: entry.resource.name,\n partition: entry.partitionValue,\n});\n\nconst rejected = (entry: PublisherEntry, reason: string, rejectedRows: number): PublishAck => ({\n status: \"rejected\",\n resource: entry.resource.name,\n reason,\n rejectedRows,\n partition: entry.partitionValue,\n});\n\nexport const layerWings = (\n config: WingsPublisherConfig,\n): Layer.Layer<Publisher, ConnectorError, Wings.WingsClient.WingsClient> =>\n Layer.effect(Publisher)(\n Effect.gen(function* () {\n const entries = new Map<string, PublisherEntry>();\n const clusterClient = yield* Wings.WingsClient.clusterClient;\n\n for (const resource of config.connector.resources) {\n const mappingInput = config.tables[resource.name];\n if (!mappingInput) {\n return yield* Effect.fail(\n new ConnectorError({ message: `Missing table for ${resource.name}` }),\n );\n }\n\n const mapping = normalizeMapping(mappingInput);\n const table = yield* clusterClient.getTable({ name: mapping.name }).pipe(\n Effect.mapError(\n (error) =>\n new ConnectorError({\n message: `Failed to resolve Wings table for ${resource.name}`,\n cause: error,\n }),\n ),\n );\n const { keyField, versionField } = yield* validateTableMapping({\n resource,\n table,\n partitionValue: mapping.partitionValue,\n });\n\n const publisher = yield* Wings.WingsClient.publisher({\n table,\n partitionValue: mapping.partitionValue,\n }).pipe(\n Effect.mapError((error) => new ConnectorError({ message: error.message, cause: error })),\n );\n\n entries.set(resource.name, {\n resource,\n table,\n publisher,\n partitionValue: mapping.partitionValue,\n keyField,\n versionField,\n });\n }\n\n const pushRows = (\n entry: PublisherEntry,\n operation: \"upsert\" | \"delete\",\n rows: ReadonlyArray<Rows>,\n ) =>\n Effect.gen(function* () {\n if (rows.length === 0) return accepted(entry);\n const batch = yield* buildRecordBatch(rows);\n return yield* entry.publisher\n .push({ operation, batch, partitionValue: entry.partitionValue })\n .pipe(\n Effect.matchEffect({\n onFailure: (error) => Effect.succeed(rejected(entry, error.message, rows.length)),\n onSuccess: () => Effect.succeed(accepted(entry)),\n }),\n );\n });\n\n const service: PublisherService = {\n publish: Effect.fn(SpanName.publish, { kind: \"producer\" })(function* ({\n resource,\n source: _source,\n batch,\n }) {\n const entry = entries.get(resource);\n if (!entry) {\n return yield* Effect.fail(\n new ConnectorError({ message: `Unknown resource ${resource}` }),\n );\n }\n\n if (batch.mutations.length === 0) return accepted(entry);\n\n const upserts: Rows[] = [];\n const deletes: Rows[] = [];\n for (const mutation of batch.mutations) {\n if (mutation.op === \"upsert\") {\n upserts.push(mutation.row as Rows);\n } else {\n deletes.push({\n [entry.keyField.name]: mutation.key,\n [entry.versionField.name]: mutation.version,\n });\n }\n }\n\n const upsertAck = yield* pushRows(entry, \"upsert\", upserts);\n if (upsertAck.status === \"rejected\") return upsertAck;\n\n return yield* pushRows(entry, \"delete\", deletes);\n }),\n };\n\n return Publisher.of(service);\n }),\n );\n","export type { ResourceBatch } from \"../core/types\";\nexport type { PublishAck, PublishOptions, PublisherService, PublishSource } from \"./service\";\nexport type { WingsPublisherConfig } from \"./wings\";\n\nexport { layerConsole } from \"./console\";\nexport { Publisher } from \"./service\";\nexport { layerWings } from \"./wings\";\n"],"mappings":";;;;;;;AAIA,MAAa,eAAe,MAAM,QAAQ,UAAU,CAAC,EACnD,UAAU,EAAE,UAAU,QAAQ,YAC5B,OAAO,IAAI,aAAa;AACtB,QAAO,OAAO,QAAQ,0BAA0B,OAAO,eAAe,WAAW,CAAC,KAChF,OAAO,aAAa;EAClB,WAAW,MAAM,UAAU;EAC3B,QAAQ,MAAM;EACd;EACD,CAAC,CACH;AAED,QAAO;EAAE,QAAQ;EAAqB;EAAU;EAChD,EACL,CAAC;;;;ACaF,MAAM,oBAAoB,YACxB,OAAO,YAAY,WAAW,EAAE,MAAM,SAAS,GAAG;AAEpD,MAAM,oBAAoB,SAA8B;CAEtD,MAAM,CAAC,SADO,MAAM,MAAM,cAAc,MAAM,KAAK,KAAK,CAAC,CACnC;AACtB,QAAO,QACH,OAAO,QAAQ,MAAM,GACrB,OAAO,KAAK,IAAI,eAAe,EAAE,SAAS,sBAAsB,CAAC,CAAC;;AAGxE,MAAM,wBAAwB,YAK5B,OAAO,IAAI;CACT,WAAW;EACT,MAAM,WAAW,MAAM,WAAW,YAAY,QAAQ,MAAM;EAC5D,MAAM,eAAe,MAAM,WAAW,gBAAgB,QAAQ,MAAM;EACpE,MAAM,iBAAiB,MAAM,WAAW,kBAAkB,QAAQ,MAAM;AAExE,MAAI,SAAS,SAAS,QAAQ,SAAS,IACrC,OAAM,IAAI,MACR,YAAY,QAAQ,SAAS,KAAK,QAAQ,QAAQ,SAAS,IAAI,oCAAoC,SAAS,KAAK,GAClH;AAEH,MAAI,aAAa,SAAS,QAAQ,SAAS,QACzC,OAAM,IAAI,MACR,YAAY,QAAQ,SAAS,KAAK,YAAY,QAAQ,SAAS,QAAQ,wCAAwC,aAAa,KAAK,GAClI;AAEH,MAAI,QAAQ,SAAS,WAAW,aAAa,QAAQ,CAAC,QAAQ,eAC5D,OAAM,IAAI,MAAM,YAAY,QAAQ,SAAS,KAAK,6BAA6B;AAEjF,MAAI,kBAAkB,CAAC,QAAQ,eAC7B,OAAM,IAAI,MACR,eAAe,QAAQ,MAAM,KAAK,qDACnC;AAEH,MAAI,CAAC,kBAAkB,QAAQ,eAC7B,OAAM,IAAI,MACR,eAAe,QAAQ,MAAM,KAAK,wDACnC;AAGH,SAAO;GAAE;GAAU;GAAc;;CAEnC,QAAQ,UACN,IAAI,eAAe;EACjB,SAAS,mCAAmC,QAAQ,SAAS;EAC7D;EACD,CAAC;CACL,CAAC;AAEJ,MAAM,YAAY,WAAuC;CACvD,QAAQ;CACR,UAAU,MAAM,SAAS;CACzB,WAAW,MAAM;CAClB;AAED,MAAM,YAAY,OAAuB,QAAgB,kBAAsC;CAC7F,QAAQ;CACR,UAAU,MAAM,SAAS;CACzB;CACA;CACA,WAAW,MAAM;CAClB;AAED,MAAa,cACX,WAEA,MAAM,OAAO,UAAU,CACrB,OAAO,IAAI,aAAa;CACtB,MAAM,0BAAU,IAAI,KAA6B;CACjD,MAAM,gBAAgB,OAAO,MAAM,YAAY;AAE/C,MAAK,MAAM,YAAY,OAAO,UAAU,WAAW;EACjD,MAAM,eAAe,OAAO,OAAO,SAAS;AAC5C,MAAI,CAAC,aACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,qBAAqB,SAAS,QAAQ,CAAC,CACtE;EAGH,MAAM,UAAU,iBAAiB,aAAa;EAC9C,MAAM,QAAQ,OAAO,cAAc,SAAS,EAAE,MAAM,QAAQ,MAAM,CAAC,CAAC,KAClE,OAAO,UACJ,UACC,IAAI,eAAe;GACjB,SAAS,qCAAqC,SAAS;GACvD,OAAO;GACR,CAAC,CACL,CACF;EACD,MAAM,EAAE,UAAU,iBAAiB,OAAO,qBAAqB;GAC7D;GACA;GACA,gBAAgB,QAAQ;GACzB,CAAC;EAEF,MAAM,YAAY,OAAO,MAAM,YAAY,UAAU;GACnD;GACA,gBAAgB,QAAQ;GACzB,CAAC,CAAC,KACD,OAAO,UAAU,UAAU,IAAI,eAAe;GAAE,SAAS,MAAM;GAAS,OAAO;GAAO,CAAC,CAAC,CACzF;AAED,UAAQ,IAAI,SAAS,MAAM;GACzB;GACA;GACA;GACA,gBAAgB,QAAQ;GACxB;GACA;GACD,CAAC;;CAGJ,MAAM,YACJ,OACA,WACA,SAEA,OAAO,IAAI,aAAa;AACtB,MAAI,KAAK,WAAW,EAAG,QAAO,SAAS,MAAM;EAC7C,MAAM,QAAQ,OAAO,iBAAiB,KAAK;AAC3C,SAAO,OAAO,MAAM,UACjB,KAAK;GAAE;GAAW;GAAO,gBAAgB,MAAM;GAAgB,CAAC,CAChE,KACC,OAAO,YAAY;GACjB,YAAY,UAAU,OAAO,QAAQ,SAAS,OAAO,MAAM,SAAS,KAAK,OAAO,CAAC;GACjF,iBAAiB,OAAO,QAAQ,SAAS,MAAM,CAAC;GACjD,CAAC,CACH;GACH;CAEJ,MAAMA,UAA4B,EAChC,SAAS,OAAO,GAAG,SAAS,SAAS,EAAE,MAAM,YAAY,CAAC,CAAC,WAAW,EACpE,UACA,QAAQ,SACR,SACC;EACD,MAAM,QAAQ,QAAQ,IAAI,SAAS;AACnC,MAAI,CAAC,MACH,QAAO,OAAO,OAAO,KACnB,IAAI,eAAe,EAAE,SAAS,oBAAoB,YAAY,CAAC,CAChE;AAGH,MAAI,MAAM,UAAU,WAAW,EAAG,QAAO,SAAS,MAAM;EAExD,MAAMC,UAAkB,EAAE;EAC1B,MAAMC,UAAkB,EAAE;AAC1B,OAAK,MAAM,YAAY,MAAM,UAC3B,KAAI,SAAS,OAAO,SAClB,SAAQ,KAAK,SAAS,IAAY;MAElC,SAAQ,KAAK;IACV,MAAM,SAAS,OAAO,SAAS;IAC/B,MAAM,aAAa,OAAO,SAAS;GACrC,CAAC;EAIN,MAAM,YAAY,OAAO,SAAS,OAAO,UAAU,QAAQ;AAC3D,MAAI,UAAU,WAAW,WAAY,QAAO;AAE5C,SAAO,OAAO,SAAS,OAAO,UAAU,QAAQ;GAChD,EACH;AAED,QAAO,UAAU,GAAG,QAAQ;EAC5B,CACH"}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
import { a as RuntimeConfigError, i as PlatformRuntimeKey, n as layerHosted, o as RuntimeConfigErrorCode, r as PlatformRuntimeDefault } from "../index-o1-Z3n7V.js";
|
|
2
|
+
export { PlatformRuntimeDefault, PlatformRuntimeKey, RuntimeConfigError, RuntimeConfigErrorCode, layerHosted };
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { n as PlatformRuntimeKey, t as PlatformRuntimeDefault } from "../constants-BRhIX_rz.js";
|
|
2
|
+
import { n as layerHosted, r as RuntimeConfigError } from "../runtime-config-BpB-d_bF.js";
|
|
3
|
+
|
|
4
|
+
export { PlatformRuntimeDefault, PlatformRuntimeKey, RuntimeConfigError, layerHosted };
|