@useairfoil/connector-kit 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/METRICS.md +78 -0
- package/README.md +190 -21
- package/dist/auth/index.d.ts +2 -0
- package/dist/auth/index.js +3 -0
- package/dist/auth-DvU_CZ7C.js +42 -0
- package/dist/auth-DvU_CZ7C.js.map +1 -0
- package/dist/connector-app/index.d.ts +14 -9
- package/dist/connector-app/index.js +9 -6
- package/dist/connector-app-d7Z-SZLQ.js +64 -0
- package/dist/connector-app-d7Z-SZLQ.js.map +1 -0
- package/dist/constants-BRhIX_rz.js +27 -0
- package/dist/constants-BRhIX_rz.js.map +1 -0
- package/dist/errors/index.d.ts +1 -1
- package/dist/{index-DK9YcR6W.d.ts → index-Byr0Lf0C.d.ts} +4 -4
- package/dist/index-CBT-JX-T.d.ts +173 -0
- package/dist/index-CTHVudvQ.d.ts +41 -0
- package/dist/index-CTLYQ74P.d.ts +80 -0
- package/dist/index-Coh_SkBs.d.ts +95 -0
- package/dist/{index-DyuT2jzP.d.ts → index-D54ySAKn.d.ts} +1 -1
- package/dist/{index-BlhJCpEk.d.ts → index-DtJX_tic.d.ts} +39 -10
- package/dist/{index-8KnVpoZp.d.ts → index-J42AZaXU.d.ts} +11 -5
- package/dist/{index-CQJsFJs3.d.ts → index-fIIBVAai.d.ts} +6 -4
- package/dist/index-o1-Z3n7V.d.ts +55 -0
- package/dist/index.d.ts +14 -9
- package/dist/index.js +14 -8
- package/dist/index.js.map +1 -1
- package/dist/ingestion/index.d.ts +5 -5
- package/dist/ingestion/index.js +7 -4
- package/dist/ingestion-BQo4cbGw.js +200 -0
- package/dist/ingestion-BQo4cbGw.js.map +1 -0
- package/dist/manifest/index.d.ts +2 -0
- package/dist/manifest/index.js +4 -0
- package/dist/manifest-ByE6YPG0.js +221 -0
- package/dist/manifest-ByE6YPG0.js.map +1 -0
- package/dist/metrics-CVAk1ALR.d.ts +53 -0
- package/dist/metrics-CWPy2Aqc.js +96 -0
- package/dist/metrics-CWPy2Aqc.js.map +1 -0
- package/dist/metrics.d.ts +4 -0
- package/dist/metrics.js +3 -0
- package/dist/publisher/index.d.ts +4 -4
- package/dist/publisher/index.js +3 -2
- package/dist/{publisher-DusgYvaV.js → publisher-BYT3RLVO.js} +2 -2
- package/dist/{publisher-DusgYvaV.js.map → publisher-BYT3RLVO.js.map} +1 -1
- package/dist/runtime-config/index.d.ts +2 -0
- package/dist/runtime-config/index.js +4 -0
- package/dist/runtime-config-BpB-d_bF.js +51 -0
- package/dist/runtime-config-BpB-d_bF.js.map +1 -0
- package/dist/server-CXTQ7odl.js +183 -0
- package/dist/server-CXTQ7odl.js.map +1 -0
- package/dist/{telemetry-p0dE5Mdu.js → service-DuFiQwid.js} +30 -10
- package/dist/service-DuFiQwid.js.map +1 -0
- package/dist/{service-B7g_lHMI.d.ts → service-IE5qaf9x.d.ts} +3 -3
- package/dist/state-store/index.d.ts +3 -4
- package/dist/state-store/index.js +4 -2
- package/dist/state-store-yl6LiTQ0.js +198 -0
- package/dist/state-store-yl6LiTQ0.js.map +1 -0
- package/dist/status/index.d.ts +5 -0
- package/dist/status/index.js +6 -0
- package/dist/status-B-oZBc33.js +84 -0
- package/dist/status-B-oZBc33.js.map +1 -0
- package/dist/{types-ZNrA-Dn0.d.ts → types-B35J5YPu.d.ts} +20 -6
- package/dist/webhook/index.d.ts +5 -4
- package/dist/webhook/index.js +5 -3
- package/dist/{webhook-BSA-tXtU.js → webhook-DI-Afxkg.js} +2 -2
- package/dist/{webhook-BSA-tXtU.js.map → webhook-DI-Afxkg.js.map} +1 -1
- package/package.json +36 -7
- package/src/auth/index.ts +1 -0
- package/src/auth/token-cache.ts +54 -0
- package/src/connector-app/check.ts +76 -0
- package/src/connector-app/index.ts +1 -0
- package/src/connector-app/start.ts +4 -0
- package/src/core/builder.ts +8 -3
- package/src/core/types.ts +20 -2
- package/src/index.ts +5 -0
- package/src/ingestion/engine.ts +246 -133
- package/src/manifest/config.ts +232 -0
- package/src/manifest/index.ts +14 -0
- package/src/manifest/manifest.ts +31 -0
- package/src/manifest/number-schema.ts +17 -0
- package/src/manifest/schema.ts +160 -0
- package/src/metrics.ts +128 -0
- package/src/publisher/instrumented.ts +68 -0
- package/src/runtime-config/constants.ts +27 -0
- package/src/runtime-config/error.ts +13 -0
- package/src/runtime-config/index.ts +3 -0
- package/src/runtime-config/layer.ts +56 -0
- package/src/state-store/index.ts +2 -1
- package/src/state-store/keys.ts +22 -0
- package/src/state-store/layer-key-value-store.ts +132 -0
- package/src/state-store/layer-memory.ts +11 -0
- package/src/state-store/layer-sql.ts +50 -0
- package/src/state-store/schema.ts +57 -0
- package/src/state-store/service.ts +39 -6
- package/src/state-store/state.ts +17 -0
- package/src/status/index.ts +2 -0
- package/src/status/route.ts +77 -0
- package/src/status/types.ts +33 -0
- package/src/telemetry.ts +46 -4
- package/src/webhook/server.ts +54 -19
- package/dist/connector-app-CvqbfN6R.js +0 -32
- package/dist/connector-app-CvqbfN6R.js.map +0 -1
- package/dist/index-DaNW63Mv.d.ts +0 -21
- package/dist/ingestion-CMCroK7Y.js +0 -171
- package/dist/ingestion-CMCroK7Y.js.map +0 -1
- package/dist/server-DAeFWT4o.js +0 -113
- package/dist/server-DAeFWT4o.js.map +0 -1
- package/dist/state-store-DG_4QpWG.js +0 -31
- package/dist/state-store-DG_4QpWG.js.map +0 -1
- package/dist/telemetry-p0dE5Mdu.js.map +0 -1
- package/src/state-store/memory.ts +0 -25
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { Config, Option, Redacted, Schema, Struct } from "effect";
|
|
2
|
+
|
|
3
|
+
import { isPlatformRuntimeKey } from "../runtime-config/constants";
|
|
4
|
+
import { makeNumberSchema, type NumberConstraints } from "./number-schema";
|
|
5
|
+
|
|
6
|
+
type ConfigFieldSpecBase = {
|
|
7
|
+
readonly runtimeKey: string;
|
|
8
|
+
readonly required: boolean;
|
|
9
|
+
readonly description?: string;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
type ConfigFieldDefinition = ConfigFieldSpecBase &
|
|
13
|
+
(
|
|
14
|
+
| {
|
|
15
|
+
readonly type: "string";
|
|
16
|
+
readonly secret: false;
|
|
17
|
+
readonly default?: string;
|
|
18
|
+
readonly values?: never;
|
|
19
|
+
}
|
|
20
|
+
| {
|
|
21
|
+
readonly type: "string";
|
|
22
|
+
readonly secret: true;
|
|
23
|
+
readonly default?: never;
|
|
24
|
+
readonly values?: never;
|
|
25
|
+
}
|
|
26
|
+
| {
|
|
27
|
+
readonly type: "number";
|
|
28
|
+
readonly secret: false;
|
|
29
|
+
readonly default?: number;
|
|
30
|
+
readonly values?: never;
|
|
31
|
+
readonly integer?: boolean;
|
|
32
|
+
readonly minimum?: number;
|
|
33
|
+
}
|
|
34
|
+
| {
|
|
35
|
+
readonly type: "boolean";
|
|
36
|
+
readonly secret: false;
|
|
37
|
+
readonly default?: boolean;
|
|
38
|
+
readonly values?: never;
|
|
39
|
+
}
|
|
40
|
+
| {
|
|
41
|
+
readonly type: "select";
|
|
42
|
+
readonly secret: false;
|
|
43
|
+
readonly default?: string;
|
|
44
|
+
readonly values: readonly [string, ...Array<string>];
|
|
45
|
+
}
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
/** Serializable, browser-safe metadata for one user-provided connector setting. */
|
|
49
|
+
export type ConfigFieldSpec = ConfigFieldDefinition & { readonly name: string };
|
|
50
|
+
|
|
51
|
+
/** Pairs an Effect runtime config with the metadata used to generate its form field. */
|
|
52
|
+
export type ConfigField<A> = {
|
|
53
|
+
readonly config: Config.Config<A>;
|
|
54
|
+
readonly spec: ConfigFieldDefinition;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
type FieldBaseOptions = {
|
|
58
|
+
readonly runtimeKey: string;
|
|
59
|
+
readonly description?: string;
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
type FieldOptions<A extends string | number | boolean = string | number | boolean> =
|
|
63
|
+
FieldBaseOptions &
|
|
64
|
+
(
|
|
65
|
+
| {
|
|
66
|
+
readonly required?: true;
|
|
67
|
+
readonly default?: A;
|
|
68
|
+
}
|
|
69
|
+
| {
|
|
70
|
+
readonly required: false;
|
|
71
|
+
readonly default: A;
|
|
72
|
+
}
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
type NumberFieldOptions = FieldOptions<number> & NumberConstraints;
|
|
76
|
+
|
|
77
|
+
type FieldsRecord = Record<string, ConfigField<unknown>>;
|
|
78
|
+
|
|
79
|
+
/** Preserves each field's value type while `Struct.map` extracts its Effect Config. */
|
|
80
|
+
interface ConfigFieldToConfig extends Struct.Lambda {
|
|
81
|
+
<A>(field: ConfigField<A>): Config.Config<A>;
|
|
82
|
+
readonly "~lambda.out": this["~lambda.in"] extends ConfigField<infer A>
|
|
83
|
+
? Config.Config<A>
|
|
84
|
+
: never;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const configFieldToConfig = Struct.lambda<ConfigFieldToConfig>((field) => field.config);
|
|
88
|
+
|
|
89
|
+
type ConfigOf<Fields extends FieldsRecord> = {
|
|
90
|
+
readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A> ? A : never;
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
/** Effect config and serializable manifest metadata produced from one field definition. */
|
|
94
|
+
export type ConnectorConfigDef<Fields extends FieldsRecord> = {
|
|
95
|
+
readonly fields: {
|
|
96
|
+
readonly [Key in keyof Fields]: Fields[Key] extends ConfigField<infer A>
|
|
97
|
+
? Config.Config<A>
|
|
98
|
+
: never;
|
|
99
|
+
};
|
|
100
|
+
readonly config: Config.Config<ConfigOf<Fields>>;
|
|
101
|
+
readonly spec: ReadonlyArray<ConfigFieldSpec>;
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
export type ConfigValuesOf<Def extends ConnectorConfigDef<FieldsRecord>> =
|
|
105
|
+
Def extends ConnectorConfigDef<infer Fields> ? ConfigOf<Fields> : never;
|
|
106
|
+
|
|
107
|
+
const withDefault = <A extends string | number | boolean>(
|
|
108
|
+
config: Config.Config<A>,
|
|
109
|
+
defaultValue: A | undefined,
|
|
110
|
+
) => (defaultValue === undefined ? config : Config.withDefault(config, defaultValue));
|
|
111
|
+
|
|
112
|
+
/** Defines a required non-empty string, optionally with a default. */
|
|
113
|
+
export const string = (options: FieldOptions<string>): ConfigField<string> => ({
|
|
114
|
+
config: withDefault(Config.nonEmptyString(options.runtimeKey), options.default),
|
|
115
|
+
spec: {
|
|
116
|
+
runtimeKey: options.runtimeKey,
|
|
117
|
+
type: "string",
|
|
118
|
+
required: options.required ?? true,
|
|
119
|
+
secret: false,
|
|
120
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
121
|
+
...(options.default !== undefined ? { default: options.default } : {}),
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
/** Defines a finite number with optional integer, minimum, and default constraints. */
|
|
126
|
+
export const number = (options: NumberFieldOptions): ConfigField<number> => {
|
|
127
|
+
const schema = makeNumberSchema(options);
|
|
128
|
+
const defaultValue =
|
|
129
|
+
options.default === undefined ? undefined : Schema.decodeUnknownSync(schema)(options.default);
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
config: withDefault(Config.schema(schema, options.runtimeKey), defaultValue),
|
|
133
|
+
spec: {
|
|
134
|
+
runtimeKey: options.runtimeKey,
|
|
135
|
+
type: "number",
|
|
136
|
+
required: options.required ?? true,
|
|
137
|
+
secret: false,
|
|
138
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
139
|
+
...(defaultValue !== undefined ? { default: defaultValue } : {}),
|
|
140
|
+
...(options.integer !== undefined ? { integer: options.integer } : {}),
|
|
141
|
+
...(options.minimum !== undefined ? { minimum: options.minimum } : {}),
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
/** Defines a boolean, optionally with a default. */
|
|
147
|
+
export const boolean = (options: FieldOptions<boolean>): ConfigField<boolean> => ({
|
|
148
|
+
config: withDefault(Config.boolean(options.runtimeKey), options.default),
|
|
149
|
+
spec: {
|
|
150
|
+
runtimeKey: options.runtimeKey,
|
|
151
|
+
type: "boolean",
|
|
152
|
+
required: options.required ?? true,
|
|
153
|
+
secret: false,
|
|
154
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
155
|
+
...(options.default !== undefined ? { default: options.default } : {}),
|
|
156
|
+
},
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
/** Defines a string literal choice whose values are preserved in the inferred type. */
|
|
160
|
+
export const select = <const Values extends readonly [string, ...Array<string>]>(
|
|
161
|
+
options: FieldOptions<Values[number]> & { readonly values: Values },
|
|
162
|
+
): ConfigField<Values[number]> => ({
|
|
163
|
+
config: withDefault(Config.literals(options.values, options.runtimeKey), options.default),
|
|
164
|
+
spec: {
|
|
165
|
+
runtimeKey: options.runtimeKey,
|
|
166
|
+
type: "select",
|
|
167
|
+
required: options.required ?? true,
|
|
168
|
+
secret: false,
|
|
169
|
+
values: options.values,
|
|
170
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
171
|
+
...(options.default !== undefined ? { default: options.default } : {}),
|
|
172
|
+
},
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
/** Defines a required, non-empty secret decoded as `Redacted<string>`. */
|
|
176
|
+
export const secret = (
|
|
177
|
+
options: FieldBaseOptions & { readonly required?: true },
|
|
178
|
+
): ConfigField<Redacted.Redacted<string>> => ({
|
|
179
|
+
config: Config.schema(Schema.Redacted(Schema.NonEmptyString), options.runtimeKey),
|
|
180
|
+
spec: {
|
|
181
|
+
runtimeKey: options.runtimeKey,
|
|
182
|
+
type: "string",
|
|
183
|
+
required: options.required ?? true,
|
|
184
|
+
secret: true,
|
|
185
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
186
|
+
},
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
/** Makes an existing field optional and represents a missing value as `Option.none()`. */
|
|
190
|
+
export const optional = <A>(field: ConfigField<A>): ConfigField<Option.Option<A>> => ({
|
|
191
|
+
config: Config.option(field.config),
|
|
192
|
+
spec: {
|
|
193
|
+
...field.spec,
|
|
194
|
+
required: false,
|
|
195
|
+
},
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Defines user-facing connector config once for both Effect runtime loading and
|
|
200
|
+
* frontend form generation. The object keys become logical form/API field names;
|
|
201
|
+
* `runtimeKey` remains the key read by Effect Config.
|
|
202
|
+
*
|
|
203
|
+
* @example
|
|
204
|
+
* ```ts
|
|
205
|
+
* const config = define({
|
|
206
|
+
* apiUrl: string({ runtimeKey: "API_URL" }),
|
|
207
|
+
* token: secret({ runtimeKey: "API_TOKEN" }),
|
|
208
|
+
* });
|
|
209
|
+
*
|
|
210
|
+
* const RuntimeConfig = config.config;
|
|
211
|
+
* const formFields = config.spec;
|
|
212
|
+
* ```
|
|
213
|
+
*/
|
|
214
|
+
export const define = <const Fields extends FieldsRecord>(
|
|
215
|
+
fields: Fields,
|
|
216
|
+
): ConnectorConfigDef<Fields> => {
|
|
217
|
+
const spec = Object.entries(fields).map(([name, field]) => ({ name, ...field.spec }));
|
|
218
|
+
const platformField = spec.find((field) => isPlatformRuntimeKey(field.runtimeKey));
|
|
219
|
+
if (platformField !== undefined) {
|
|
220
|
+
throw new Error(`Connector runtime key is platform-owned: ${platformField.runtimeKey}`);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const configFields = Struct.map(fields, configFieldToConfig);
|
|
224
|
+
|
|
225
|
+
return {
|
|
226
|
+
fields: configFields,
|
|
227
|
+
// Effect's recursive Config.Wrap cannot reduce this generic mapped type,
|
|
228
|
+
// although ConfigFieldToConfig preserves the value type of every property.
|
|
229
|
+
config: Config.unwrap(configFields as Config.Wrap<ConfigOf<Fields>>),
|
|
230
|
+
spec,
|
|
231
|
+
};
|
|
232
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export {
|
|
2
|
+
boolean,
|
|
3
|
+
define as defineConfig,
|
|
4
|
+
number,
|
|
5
|
+
optional,
|
|
6
|
+
secret,
|
|
7
|
+
select,
|
|
8
|
+
string,
|
|
9
|
+
} from "./config";
|
|
10
|
+
export type { ConfigField, ConfigFieldSpec, ConfigValuesOf, ConnectorConfigDef } from "./config";
|
|
11
|
+
export { define } from "./manifest";
|
|
12
|
+
export type { ConnectorManifest, ResourceCapability } from "./manifest";
|
|
13
|
+
export { configSchema, decodeConfig, toRuntimeDocument } from "./schema";
|
|
14
|
+
export type { ConnectorConfigSchema, ConnectorConfigValues, RuntimeConfigDocument } from "./schema";
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ConfigFieldSpec } from "./config";
|
|
2
|
+
|
|
3
|
+
export type ResourceCapability = "backfill" | "changes" | "webhook";
|
|
4
|
+
|
|
5
|
+
/** Browser-safe connector metadata consumed by the platform and runtime adapters. */
|
|
6
|
+
export type ConnectorManifest = {
|
|
7
|
+
readonly name: string;
|
|
8
|
+
readonly title: string;
|
|
9
|
+
readonly description?: string;
|
|
10
|
+
readonly config: ReadonlyArray<ConfigFieldSpec>;
|
|
11
|
+
readonly resources: ReadonlyArray<{
|
|
12
|
+
readonly name: string;
|
|
13
|
+
readonly capabilities: ReadonlyArray<ResourceCapability>;
|
|
14
|
+
}>;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Defines browser-safe connector metadata while preserving literal names and capabilities.
|
|
19
|
+
*
|
|
20
|
+
* @example
|
|
21
|
+
* ```ts
|
|
22
|
+
* const manifest = define({
|
|
23
|
+
* name: "producer-example",
|
|
24
|
+
* title: "Example",
|
|
25
|
+
* config: ExampleConfig.spec,
|
|
26
|
+
* resources: [{ name: "orders", capabilities: ["backfill"] }],
|
|
27
|
+
* });
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
30
|
+
export const define = <const Manifest extends ConnectorManifest>(manifest: Manifest): Manifest =>
|
|
31
|
+
manifest;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
|
|
3
|
+
export type NumberConstraints = {
|
|
4
|
+
readonly integer?: boolean;
|
|
5
|
+
readonly minimum?: number;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export const makeNumberSchema = (constraints: NumberConstraints) => {
|
|
9
|
+
if (constraints.integer) {
|
|
10
|
+
return constraints.minimum === undefined
|
|
11
|
+
? Schema.Int
|
|
12
|
+
: Schema.Int.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
|
|
13
|
+
}
|
|
14
|
+
return constraints.minimum === undefined
|
|
15
|
+
? Schema.Finite
|
|
16
|
+
: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(constraints.minimum));
|
|
17
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { Predicate, Schema, SchemaTransformation, type Types } from "effect";
|
|
2
|
+
|
|
3
|
+
import type { ConfigFieldSpec } from "./config";
|
|
4
|
+
import type { ConnectorManifest } from "./manifest";
|
|
5
|
+
|
|
6
|
+
import { makeNumberSchema } from "./number-schema";
|
|
7
|
+
|
|
8
|
+
type FieldValueOf<Field extends ConfigFieldSpec> = Field["type"] extends "number"
|
|
9
|
+
? number
|
|
10
|
+
: Field["type"] extends "boolean"
|
|
11
|
+
? boolean
|
|
12
|
+
: Field["values"] extends ReadonlyArray<string>
|
|
13
|
+
? Field["values"][number]
|
|
14
|
+
: string;
|
|
15
|
+
|
|
16
|
+
export type ConnectorConfigValues<Manifest extends ConnectorManifest> = Types.Simplify<
|
|
17
|
+
{
|
|
18
|
+
readonly [Field in Manifest["config"][number] as Field["required"] extends true
|
|
19
|
+
? Field["name"]
|
|
20
|
+
: never]: FieldValueOf<Field>;
|
|
21
|
+
} & {
|
|
22
|
+
readonly [Field in Manifest["config"][number] as Field["required"] extends true
|
|
23
|
+
? never
|
|
24
|
+
: Field["name"]]?: FieldValueOf<Field> | undefined;
|
|
25
|
+
}
|
|
26
|
+
> &
|
|
27
|
+
object;
|
|
28
|
+
|
|
29
|
+
export type ConnectorConfigSchema<Manifest extends ConnectorManifest> = Schema.Codec<
|
|
30
|
+
ConnectorConfigValues<Manifest>,
|
|
31
|
+
unknown
|
|
32
|
+
>;
|
|
33
|
+
|
|
34
|
+
/** Flat primitive document keyed by the runtime keys consumed by Effect Config. */
|
|
35
|
+
export type RuntimeConfigDocument = Readonly<Record<string, string | number | boolean>>;
|
|
36
|
+
|
|
37
|
+
// Browser forms submit numbers and booleans as strings, while API callers may
|
|
38
|
+
// already send their canonical JSON representations. These schemas accept both.
|
|
39
|
+
const numberFromForm = (field: Extract<ConfigFieldSpec, { readonly type: "number" }>) => {
|
|
40
|
+
const numberSchema = makeNumberSchema(field);
|
|
41
|
+
return Schema.Union([
|
|
42
|
+
numberSchema,
|
|
43
|
+
Schema.NonEmptyString.pipe(
|
|
44
|
+
Schema.decodeTo(numberSchema, SchemaTransformation.numberFromString),
|
|
45
|
+
),
|
|
46
|
+
]);
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const BooleanFromForm = Schema.Union([
|
|
50
|
+
Schema.Boolean,
|
|
51
|
+
Schema.Literals(["true", "false"]).pipe(
|
|
52
|
+
Schema.decodeTo(
|
|
53
|
+
Schema.Boolean,
|
|
54
|
+
SchemaTransformation.transform({
|
|
55
|
+
decode: (value) => value === "true",
|
|
56
|
+
encode: (value) => (value ? "true" : "false"),
|
|
57
|
+
}),
|
|
58
|
+
),
|
|
59
|
+
),
|
|
60
|
+
]);
|
|
61
|
+
|
|
62
|
+
// An empty optional form control means "not supplied", not an empty config value.
|
|
63
|
+
const EmptyStringAsUndefined = Schema.Literal("").pipe(
|
|
64
|
+
Schema.decodeTo(
|
|
65
|
+
Schema.Undefined,
|
|
66
|
+
SchemaTransformation.transform({ decode: () => undefined, encode: () => "" }),
|
|
67
|
+
),
|
|
68
|
+
);
|
|
69
|
+
|
|
70
|
+
const baseFieldSchema = (field: ConfigFieldSpec): Schema.Codec<unknown, unknown> => {
|
|
71
|
+
switch (field.type) {
|
|
72
|
+
case "number":
|
|
73
|
+
return numberFromForm(field);
|
|
74
|
+
case "boolean":
|
|
75
|
+
return BooleanFromForm;
|
|
76
|
+
case "select":
|
|
77
|
+
return Schema.Literals(field.values);
|
|
78
|
+
case "string":
|
|
79
|
+
return Schema.NonEmptyString;
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
const fieldSchema = (field: ConfigFieldSpec) =>
|
|
84
|
+
field.required
|
|
85
|
+
? baseFieldSchema(field)
|
|
86
|
+
: Schema.optional(Schema.Union([EmptyStringAsUndefined, baseFieldSchema(field)]));
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Builds an Effect Schema for logical form/API values from a connector manifest.
|
|
90
|
+
*
|
|
91
|
+
* @example
|
|
92
|
+
* ```ts
|
|
93
|
+
* const decode = Schema.decodeUnknownEffect(configSchema(manifest));
|
|
94
|
+
* const values = decode({
|
|
95
|
+
* apiUrl: "https://example.com",
|
|
96
|
+
* retries: "3",
|
|
97
|
+
* });
|
|
98
|
+
* ```
|
|
99
|
+
*/
|
|
100
|
+
export const configSchema = <const Manifest extends ConnectorManifest>(
|
|
101
|
+
manifest: Manifest,
|
|
102
|
+
): ConnectorConfigSchema<Manifest> => {
|
|
103
|
+
const fields = Object.fromEntries(
|
|
104
|
+
manifest.config.map((field) => [field.name, fieldSchema(field)]),
|
|
105
|
+
);
|
|
106
|
+
// Object.fromEntries cannot retain keys discovered from the runtime manifest;
|
|
107
|
+
// the public mapped type restores the same relationship for callers.
|
|
108
|
+
return Schema.Struct(fields) as unknown as ConnectorConfigSchema<Manifest>;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
/** Decodes browser/API config, applies manifest defaults, and rejects undeclared fields. */
|
|
112
|
+
export const decodeConfig = <const Manifest extends ConnectorManifest>(
|
|
113
|
+
manifest: Manifest,
|
|
114
|
+
input: unknown,
|
|
115
|
+
) => {
|
|
116
|
+
const defaults = Object.fromEntries(
|
|
117
|
+
manifest.config.flatMap((field) =>
|
|
118
|
+
field.default === undefined ? [] : [[field.name, field.default]],
|
|
119
|
+
),
|
|
120
|
+
);
|
|
121
|
+
const optionalFields = new Set(
|
|
122
|
+
manifest.config.flatMap((field) => (field.required ? [] : [field.name])),
|
|
123
|
+
);
|
|
124
|
+
const normalizedInput = Predicate.isObject(input)
|
|
125
|
+
? Object.fromEntries(
|
|
126
|
+
Object.entries(input).filter(([name, value]) => value !== "" || !optionalFields.has(name)),
|
|
127
|
+
)
|
|
128
|
+
: input;
|
|
129
|
+
const inputWithDefaults = Predicate.isObject(normalizedInput)
|
|
130
|
+
? Object.assign({}, defaults, normalizedInput)
|
|
131
|
+
: normalizedInput;
|
|
132
|
+
|
|
133
|
+
return Schema.decodeUnknownEffect(configSchema(manifest))(inputWithDefaults, {
|
|
134
|
+
onExcessProperty: "error",
|
|
135
|
+
});
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Maps validated logical values to the flat runtime-key document mounted in a pod.
|
|
140
|
+
* Missing optional values are omitted.
|
|
141
|
+
*
|
|
142
|
+
* @example
|
|
143
|
+
* ```ts
|
|
144
|
+
* toRuntimeDocument(manifest, { apiUrl: "https://example.com" });
|
|
145
|
+
* // { API_URL: "https://example.com" }
|
|
146
|
+
* ```
|
|
147
|
+
*/
|
|
148
|
+
export const toRuntimeDocument = <const Manifest extends ConnectorManifest>(
|
|
149
|
+
manifest: Manifest,
|
|
150
|
+
values: ConnectorConfigValues<Manifest>,
|
|
151
|
+
): RuntimeConfigDocument => {
|
|
152
|
+
const document: Record<string, string | number | boolean> = {};
|
|
153
|
+
for (const field of manifest.config) {
|
|
154
|
+
const value: unknown = Reflect.get(values, field.name);
|
|
155
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
156
|
+
document[field.runtimeKey] = value;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return document;
|
|
160
|
+
};
|
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { DateTime, Effect, Metric, Option } from "effect";
|
|
2
|
+
|
|
3
|
+
import type { SyncState } from "./core/types";
|
|
4
|
+
|
|
5
|
+
export type BatchSource = "backfill" | "changes" | "webhook";
|
|
6
|
+
export type BatchOutcome = "accepted" | "rejected" | "error";
|
|
7
|
+
export type ApiRetryReason = "transport" | "timeout" | "rate_limit" | "server_error";
|
|
8
|
+
export type WebhookOutcome =
|
|
9
|
+
| "ok"
|
|
10
|
+
| "read_error"
|
|
11
|
+
| "invalid_json"
|
|
12
|
+
| "invalid_payload"
|
|
13
|
+
| "rejected"
|
|
14
|
+
| "handler_error";
|
|
15
|
+
|
|
16
|
+
export type ResourceMetricAttrs = {
|
|
17
|
+
readonly connector: string;
|
|
18
|
+
readonly resource: string;
|
|
19
|
+
readonly source: BatchSource;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export type WebhookMetricAttrs = {
|
|
23
|
+
readonly connector: string;
|
|
24
|
+
readonly path: string;
|
|
25
|
+
readonly outcome: WebhookOutcome;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type SyncStateAttrs = {
|
|
29
|
+
readonly connector: string;
|
|
30
|
+
readonly resource: string;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type LastSuccessMetricAttrs = {
|
|
34
|
+
readonly connector: string;
|
|
35
|
+
readonly resource: string;
|
|
36
|
+
readonly source: "backfill" | "changes";
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
export type ApiRetryMetricAttrs = {
|
|
40
|
+
readonly connector: string;
|
|
41
|
+
readonly reason: ApiRetryReason;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
export const entitiesUpsertedTotal = Metric.counter("airfoil_connector_entities_upserted_total", {
|
|
45
|
+
description: "Total entity upsert mutations published by connector sources",
|
|
46
|
+
incremental: true,
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
export const entitiesDeletedTotal = Metric.counter("airfoil_connector_entities_deleted_total", {
|
|
50
|
+
description: "Total entity delete mutations published by connector sources",
|
|
51
|
+
incremental: true,
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
export const batchesTotal = Metric.counter("airfoil_connector_batches_total", {
|
|
55
|
+
description: "Total resource batches published by connector sources",
|
|
56
|
+
incremental: true,
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
export const batchSize = Metric.histogram("airfoil_connector_batch_size", {
|
|
60
|
+
description: "Distribution of resource mutation batch sizes",
|
|
61
|
+
boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000],
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
export const publishDuration = Metric.timer("airfoil_connector_publish_duration_ms", {
|
|
65
|
+
description: "Publisher call duration in milliseconds",
|
|
66
|
+
boundaries: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000],
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
export const webhookRequestsTotal = Metric.counter("airfoil_connector_webhook_requests_total", {
|
|
70
|
+
description: "Total webhook HTTP requests handled by connector routes",
|
|
71
|
+
incremental: true,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
export const webhookQueueDepth = Metric.gauge("airfoil_connector_webhook_queue_depth", {
|
|
75
|
+
description: "Current queued webhook batches waiting for publish",
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
export const syncState = Metric.gauge("airfoil_connector_sync_state", {
|
|
79
|
+
description: "Connector resource sync state as a Prometheus state-set gauge",
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
export const lastSuccessTimestamp = Metric.gauge(
|
|
83
|
+
"airfoil_connector_last_success_timestamp_seconds",
|
|
84
|
+
{
|
|
85
|
+
description: "Unix timestamp of the last successful durable source checkpoint",
|
|
86
|
+
},
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
export const apiRetriesTotal = Metric.counter("airfoil_connector_api_retries_total", {
|
|
90
|
+
description: "Total provider API retries performed by the connector",
|
|
91
|
+
incremental: true,
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
export const recordWebhookRequest = (attrs: WebhookMetricAttrs) =>
|
|
95
|
+
Metric.update(Metric.withAttributes(webhookRequestsTotal, attrs), 1);
|
|
96
|
+
|
|
97
|
+
export const setWebhookQueueDepth = (connector: string, depth: number) =>
|
|
98
|
+
Metric.update(Metric.withAttributes(webhookQueueDepth, { connector }), depth);
|
|
99
|
+
|
|
100
|
+
export const setLastSuccessTimestamp = (attrs: LastSuccessMetricAttrs, lastSuccessAt?: string) => {
|
|
101
|
+
const timestamp =
|
|
102
|
+
lastSuccessAt === undefined
|
|
103
|
+
? 0
|
|
104
|
+
: DateTime.make(lastSuccessAt).pipe(
|
|
105
|
+
Option.map((dateTime) => DateTime.toEpochMillis(dateTime) / 1000),
|
|
106
|
+
Option.getOrElse(() => 0),
|
|
107
|
+
);
|
|
108
|
+
return Metric.update(Metric.withAttributes(lastSuccessTimestamp, attrs), timestamp);
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
export const recordApiRetry = (attrs: ApiRetryMetricAttrs) =>
|
|
112
|
+
Metric.update(Metric.withAttributes(apiRetriesTotal, attrs), 1);
|
|
113
|
+
|
|
114
|
+
const syncStates = ["pending", "backfilling", "live", "error"] as const;
|
|
115
|
+
|
|
116
|
+
export const setSyncState = (attrs: SyncStateAttrs, current: SyncState) =>
|
|
117
|
+
Effect.forEach(
|
|
118
|
+
syncStates,
|
|
119
|
+
(state) =>
|
|
120
|
+
Metric.update(
|
|
121
|
+
Metric.withAttributes(syncState, {
|
|
122
|
+
...attrs,
|
|
123
|
+
state,
|
|
124
|
+
}),
|
|
125
|
+
state === current ? 1 : 0,
|
|
126
|
+
),
|
|
127
|
+
{ discard: true },
|
|
128
|
+
);
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { Effect, Metric } from "effect";
|
|
2
|
+
|
|
3
|
+
import type { ResourceBatch } from "../core/types";
|
|
4
|
+
|
|
5
|
+
import { ConnectorError } from "../errors";
|
|
6
|
+
import * as Metrics from "../metrics";
|
|
7
|
+
import { Attr, EventAttr, EventName, addCurrentSpanEvent } from "../telemetry";
|
|
8
|
+
import { Publisher, type PublishSource } from "./service";
|
|
9
|
+
|
|
10
|
+
export const publishBatch = Effect.fnUntraced(function* (options: {
|
|
11
|
+
readonly connector?: string;
|
|
12
|
+
readonly resource: string;
|
|
13
|
+
readonly source: PublishSource;
|
|
14
|
+
readonly batch: ResourceBatch;
|
|
15
|
+
}) {
|
|
16
|
+
const metric = {
|
|
17
|
+
connector: options.connector ?? "unknown",
|
|
18
|
+
resource: options.resource,
|
|
19
|
+
source: options.source,
|
|
20
|
+
};
|
|
21
|
+
const upserts = options.batch.mutations.filter((mutation) => mutation.op === "upsert").length;
|
|
22
|
+
const deletes = options.batch.mutations.length - upserts;
|
|
23
|
+
|
|
24
|
+
const publisher = yield* Publisher;
|
|
25
|
+
const [duration, ack] = yield* Effect.timed(
|
|
26
|
+
publisher.publish({
|
|
27
|
+
resource: options.resource,
|
|
28
|
+
source: options.source,
|
|
29
|
+
batch: options.batch,
|
|
30
|
+
}),
|
|
31
|
+
).pipe(
|
|
32
|
+
Effect.catchCause((cause) =>
|
|
33
|
+
Metric.update(
|
|
34
|
+
Metric.withAttributes(Metrics.batchesTotal, { ...metric, outcome: "error" }),
|
|
35
|
+
1,
|
|
36
|
+
).pipe(Effect.andThen(Effect.failCause(cause))),
|
|
37
|
+
),
|
|
38
|
+
);
|
|
39
|
+
|
|
40
|
+
yield* Metric.update(Metric.withAttributes(Metrics.publishDuration, metric), duration);
|
|
41
|
+
yield* Metric.update(
|
|
42
|
+
Metric.withAttributes(Metrics.batchesTotal, { ...metric, outcome: ack.status }),
|
|
43
|
+
1,
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
yield* Effect.annotateCurrentSpan({ [Attr.publisherSuccess]: ack.status === "accepted" });
|
|
47
|
+
if (ack.status === "rejected") {
|
|
48
|
+
yield* Effect.annotateCurrentSpan({ [Attr.errorPhase]: "publish" });
|
|
49
|
+
return yield* Effect.fail(
|
|
50
|
+
new ConnectorError({
|
|
51
|
+
message: `Publisher rejected batch for ${options.resource}: ${ack.reason}`,
|
|
52
|
+
}),
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
yield* Metric.update(Metric.withAttributes(Metrics.entitiesUpsertedTotal, metric), upserts);
|
|
57
|
+
yield* Metric.update(Metric.withAttributes(Metrics.entitiesDeletedTotal, metric), deletes);
|
|
58
|
+
yield* Metric.update(
|
|
59
|
+
Metric.withAttributes(Metrics.batchSize, metric),
|
|
60
|
+
options.batch.mutations.length,
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
if (options.batch.cursor !== undefined) {
|
|
64
|
+
yield* addCurrentSpanEvent(EventName.batchCheckpoint, {
|
|
65
|
+
[EventAttr.batchCursor]: options.batch.cursor,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { HashSet } from "effect";
|
|
2
|
+
|
|
3
|
+
/** Shared runtime keys owned by Airfoil rather than an individual connector. */
|
|
4
|
+
export const PlatformRuntimeKey = {
|
|
5
|
+
configPath: "AIRFOIL_CONFIG_PATH",
|
|
6
|
+
connectorInstanceId: "AIRFOIL_CONNECTOR_INSTANCE_ID",
|
|
7
|
+
stateTable: "AIRFOIL_STATE_TABLE",
|
|
8
|
+
postgresConnectionString: "POSTGRES_CONNECTION_STRING",
|
|
9
|
+
wingsHost: "WINGS_HOST",
|
|
10
|
+
wingsNamespace: "WINGS_NAMESPACE",
|
|
11
|
+
otelEnabled: "OTEL_ENABLED",
|
|
12
|
+
otelExporterOtlpEndpoint: "OTEL_EXPORTER_OTLP_ENDPOINT",
|
|
13
|
+
otelExporterOtlpHeaders: "OTEL_EXPORTER_OTLP_HEADERS",
|
|
14
|
+
otelServiceName: "OTEL_SERVICE_NAME",
|
|
15
|
+
otelServiceVersion: "OTEL_SERVICE_VERSION",
|
|
16
|
+
otelResourceAttributes: "OTEL_RESOURCE_ATTRIBUTES",
|
|
17
|
+
} as const;
|
|
18
|
+
|
|
19
|
+
/** Defaults for optional platform-owned runtime configuration. */
|
|
20
|
+
export const PlatformRuntimeDefault = {
|
|
21
|
+
stateTable: "_airfoil_connectors_state",
|
|
22
|
+
} as const;
|
|
23
|
+
|
|
24
|
+
const platformRuntimeKeys = HashSet.fromIterable(Object.values(PlatformRuntimeKey));
|
|
25
|
+
|
|
26
|
+
/** @internal */
|
|
27
|
+
export const isPlatformRuntimeKey = (key: string): boolean => HashSet.has(platformRuntimeKeys, key);
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Data } from "effect";
|
|
2
|
+
|
|
3
|
+
export type RuntimeConfigErrorCode =
|
|
4
|
+
| "CONFIG_PATH_MISSING"
|
|
5
|
+
| "CONFIG_FILE_UNREADABLE"
|
|
6
|
+
| "CONFIG_JSON_INVALID";
|
|
7
|
+
|
|
8
|
+
/** Expected hosted-config failure with a stable code and diagnostic cause. */
|
|
9
|
+
export class RuntimeConfigError extends Data.TaggedError("RuntimeConfigError")<{
|
|
10
|
+
readonly code: RuntimeConfigErrorCode;
|
|
11
|
+
readonly message: string;
|
|
12
|
+
readonly cause: Error;
|
|
13
|
+
}> {}
|