@useairfoil/connector-kit 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/LICENSE.txt +201 -0
  2. package/METRICS.md +38 -42
  3. package/README.md +111 -444
  4. package/dist/annotations-vzSzXtr-.d.mts +34 -0
  5. package/dist/arrow-flight-DDQ0ktx9.mjs +251 -0
  6. package/dist/arrow-flight-DDQ0ktx9.mjs.map +1 -0
  7. package/dist/auth/index.d.mts +2 -0
  8. package/dist/{auth-DvU_CZ7C.js → auth/index.mjs} +6 -8
  9. package/dist/auth/index.mjs.map +1 -0
  10. package/dist/connector-app/index.d.mts +2 -0
  11. package/dist/connector-app/index.mjs +2 -0
  12. package/dist/connector-app-C6_dZnhE.mjs +256 -0
  13. package/dist/connector-app-C6_dZnhE.mjs.map +1 -0
  14. package/dist/{constants-D113Ut1x.js → constants-v3UX-hf6.mjs} +5 -5
  15. package/dist/constants-v3UX-hf6.mjs.map +1 -0
  16. package/dist/demo/index.d.mts +2 -0
  17. package/dist/demo/index.mjs +2 -0
  18. package/dist/demo-DqTJ7TWI.mjs +204 -0
  19. package/dist/demo-DqTJ7TWI.mjs.map +1 -0
  20. package/dist/errors/index.d.mts +2 -0
  21. package/dist/errors/index.mjs +2 -0
  22. package/dist/{errors-DHWylqRZ.js → errors-pT0YVJU5.mjs} +2 -3
  23. package/dist/errors-pT0YVJU5.mjs.map +1 -0
  24. package/dist/iceberg/index.d.mts +3 -0
  25. package/dist/iceberg/index.mjs +3 -0
  26. package/dist/iceberg-Cj2PwlNm.mjs +30 -0
  27. package/dist/iceberg-Cj2PwlNm.mjs.map +1 -0
  28. package/dist/index-BTAbexBp.d.mts +55 -0
  29. package/dist/{index-DXa26Ry5.d.ts → index-BX8G7d41.d.mts} +1 -2
  30. package/dist/{index-VpHaZXZw.d.ts → index-Byr8NTdG.d.mts} +11 -13
  31. package/dist/index-CV6V6fyx.d.mts +79 -0
  32. package/dist/{index-C5Ma7V7t.d.ts → index-CZKx2Yce.d.mts} +72 -37
  33. package/dist/{index-DYpEMJf-.d.ts → index-CgvFomO6.d.mts} +6 -7
  34. package/dist/{index-bfjH7Yb0.d.ts → index-DeTwIUKl.d.mts} +9 -10
  35. package/dist/{index-DKoRwyFn.d.ts → index-DkVNO6ks.d.mts} +2 -5
  36. package/dist/{index-BKw7jA2a.d.ts → index-dHS59EZl.d.mts} +10 -8
  37. package/dist/{index-CR9HJDdr.d.ts → index-eFuoMZZ3.d.mts} +18 -20
  38. package/dist/{index-ZSuiZX5H.d.ts → index-fHIJJoRe.d.mts} +25 -30
  39. package/dist/index-uFa2wwYo.d.mts +27 -0
  40. package/dist/index.d.mts +16 -0
  41. package/dist/index.mjs +45 -0
  42. package/dist/index.mjs.map +1 -0
  43. package/dist/ingestion/index.d.mts +2 -0
  44. package/dist/ingestion/index.mjs +2 -0
  45. package/dist/ingestion-DoCUwm5N.mjs +275 -0
  46. package/dist/ingestion-DoCUwm5N.mjs.map +1 -0
  47. package/dist/ingestor/index.d.mts +3 -0
  48. package/dist/ingestor/index.mjs +4 -0
  49. package/dist/ingestor-DjCEzBzZ.mjs +26 -0
  50. package/dist/ingestor-DjCEzBzZ.mjs.map +1 -0
  51. package/dist/instrumented-Bq1WX2L5.mjs +29 -0
  52. package/dist/instrumented-Bq1WX2L5.mjs.map +1 -0
  53. package/dist/manifest/index.d.mts +2 -0
  54. package/dist/manifest/{index.js → index.mjs} +2 -4
  55. package/dist/{manifest-P7kBnch9.js → manifest-CABeIhAG.mjs} +14 -19
  56. package/dist/manifest-CABeIhAG.mjs.map +1 -0
  57. package/dist/{telemetry-j_nxwYfh.js → metrics-C901IIYZ.mjs} +172 -26
  58. package/dist/metrics-C901IIYZ.mjs.map +1 -0
  59. package/dist/{metrics-C4V1pm6v.d.ts → metrics-DoJ0WeKu.d.mts} +18 -13
  60. package/dist/metrics.d.mts +2 -0
  61. package/dist/metrics.mjs +2 -0
  62. package/dist/{chunk-Bp6m_JJh.js → rolldown-runtime-D7D4PA-g.mjs} +4 -4
  63. package/dist/runtime-config/index.d.mts +2 -0
  64. package/dist/runtime-config/index.mjs +3 -0
  65. package/dist/runtime-config-Bn8Fivt1.mjs +100 -0
  66. package/dist/runtime-config-Bn8Fivt1.mjs.map +1 -0
  67. package/dist/server-D1TTN0_D.mjs +144 -0
  68. package/dist/server-D1TTN0_D.mjs.map +1 -0
  69. package/dist/service-BRoy_EId.d.mts +27 -0
  70. package/dist/service-jWxxfEht.mjs +7 -0
  71. package/dist/service-jWxxfEht.mjs.map +1 -0
  72. package/dist/state-store/index.d.mts +2 -0
  73. package/dist/state-store/index.mjs +2 -0
  74. package/dist/{state-store-BdYulQOY.js → state-store-DhSrKVEH.mjs} +15 -22
  75. package/dist/state-store-DhSrKVEH.mjs.map +1 -0
  76. package/dist/status/{index.d.ts → index.d.mts} +1 -4
  77. package/dist/status/{index.js → index.mjs} +2 -6
  78. package/dist/{status-BTTd7AUR.js → status-4h6Oxh-O.mjs} +14 -16
  79. package/dist/status-4h6Oxh-O.mjs.map +1 -0
  80. package/dist/table-CxM09xf7.mjs +307 -0
  81. package/dist/table-CxM09xf7.mjs.map +1 -0
  82. package/dist/types-NtSv6gwl.d.mts +220 -0
  83. package/dist/webhook/index.d.mts +3 -0
  84. package/dist/webhook/index.mjs +12 -0
  85. package/dist/webhook/index.mjs.map +1 -0
  86. package/package.json +72 -58
  87. package/src/catalog/retry.ts +25 -0
  88. package/src/catalog/table.ts +237 -0
  89. package/src/connector-app/create-table.ts +47 -0
  90. package/src/connector-app/demo.ts +47 -0
  91. package/src/connector-app/index.ts +3 -0
  92. package/src/connector-app/setup.ts +69 -0
  93. package/src/connector-app/start.ts +1 -1
  94. package/src/core/builder.ts +15 -23
  95. package/src/core/index.ts +11 -1
  96. package/src/core/types.ts +114 -39
  97. package/src/demo/definition.ts +68 -0
  98. package/src/demo/generate.ts +167 -0
  99. package/src/demo/index.ts +3 -0
  100. package/src/demo/run.ts +172 -0
  101. package/src/iceberg/annotations.ts +49 -0
  102. package/src/iceberg/error.ts +9 -0
  103. package/src/iceberg/index.ts +3 -0
  104. package/src/iceberg/table.ts +461 -0
  105. package/src/index.ts +3 -1
  106. package/src/ingestion/engine.ts +240 -60
  107. package/src/ingestor/arrow-flight.ts +141 -0
  108. package/src/ingestor/arrow.ts +248 -0
  109. package/src/ingestor/console.ts +13 -0
  110. package/src/ingestor/index.ts +13 -0
  111. package/src/ingestor/instrumented.ts +61 -0
  112. package/src/ingestor/service.ts +30 -0
  113. package/src/manifest/config.ts +3 -3
  114. package/src/metrics.ts +66 -20
  115. package/src/runtime-config/config.ts +11 -8
  116. package/src/runtime-config/constants.ts +3 -2
  117. package/src/runtime-config/index.ts +2 -7
  118. package/src/runtime-config/layer.ts +1 -1
  119. package/src/runtime-config/table-bindings.ts +50 -136
  120. package/src/state-store/layer-key-value-store.ts +8 -3
  121. package/src/state-store/layer-memory.ts +1 -1
  122. package/src/state-store/layer-sql.ts +4 -4
  123. package/src/state-store/schema.ts +2 -2
  124. package/src/state-store/state.ts +14 -2
  125. package/src/status/route.ts +2 -2
  126. package/src/telemetry.ts +20 -28
  127. package/src/webhook/server.ts +39 -40
  128. package/dist/auth/index.d.ts +0 -2
  129. package/dist/auth/index.js +0 -3
  130. package/dist/auth-DvU_CZ7C.js.map +0 -1
  131. package/dist/connector-app/index.d.ts +0 -14
  132. package/dist/connector-app/index.js +0 -12
  133. package/dist/connector-app-BljHd0Pe.js +0 -64
  134. package/dist/connector-app-BljHd0Pe.js.map +0 -1
  135. package/dist/constants-D113Ut1x.js.map +0 -1
  136. package/dist/errors/index.d.ts +0 -2
  137. package/dist/errors/index.js +0 -3
  138. package/dist/errors-DHWylqRZ.js.map +0 -1
  139. package/dist/index-BA8ASHKr.d.ts +0 -31
  140. package/dist/index.d.ts +0 -14
  141. package/dist/index.js +0 -60
  142. package/dist/index.js.map +0 -1
  143. package/dist/ingestion/index.d.ts +0 -6
  144. package/dist/ingestion/index.js +0 -11
  145. package/dist/ingestion-QBGKrFr9.js +0 -200
  146. package/dist/ingestion-QBGKrFr9.js.map +0 -1
  147. package/dist/manifest/index.d.ts +0 -2
  148. package/dist/manifest-P7kBnch9.js.map +0 -1
  149. package/dist/metrics-inkCAxKq.js +0 -136
  150. package/dist/metrics-inkCAxKq.js.map +0 -1
  151. package/dist/metrics.d.ts +0 -4
  152. package/dist/metrics.js +0 -5
  153. package/dist/publisher/index.d.ts +0 -6
  154. package/dist/publisher/index.js +0 -8
  155. package/dist/publisher-BdWJOfer.js +0 -138
  156. package/dist/publisher-BdWJOfer.js.map +0 -1
  157. package/dist/runtime-config/index.d.ts +0 -4
  158. package/dist/runtime-config/index.js +0 -5
  159. package/dist/runtime-config-IbyCVpei.js +0 -60
  160. package/dist/runtime-config-IbyCVpei.js.map +0 -1
  161. package/dist/server-D-eLX3yN.js +0 -183
  162. package/dist/server-D-eLX3yN.js.map +0 -1
  163. package/dist/service-CUjGhe-7.d.ts +0 -32
  164. package/dist/service-DiP-XETo.js +0 -8
  165. package/dist/service-DiP-XETo.js.map +0 -1
  166. package/dist/state-store/index.d.ts +0 -3
  167. package/dist/state-store/index.js +0 -5
  168. package/dist/state-store-BdYulQOY.js.map +0 -1
  169. package/dist/status-BTTd7AUR.js.map +0 -1
  170. package/dist/table-bindings-Dp1DhTeS.js +0 -106
  171. package/dist/table-bindings-Dp1DhTeS.js.map +0 -1
  172. package/dist/telemetry-j_nxwYfh.js.map +0 -1
  173. package/dist/types-jVD5Ys43.d.ts +0 -174
  174. package/dist/webhook/index.d.ts +0 -7
  175. package/dist/webhook/index.js +0 -9
  176. package/dist/webhook-MhFcLW-o.js +0 -13
  177. package/dist/webhook-MhFcLW-o.js.map +0 -1
  178. package/src/publisher/console.ts +0 -18
  179. package/src/publisher/index.ts +0 -7
  180. package/src/publisher/instrumented.ts +0 -75
  181. package/src/publisher/service.ts +0 -37
  182. package/src/publisher/wings.ts +0 -204
@@ -0,0 +1,307 @@
1
+ import { Data, Effect, Schema } from "effect";
2
+ //#region src/iceberg/error.ts
3
+ /** A schema could not be turned into an Iceberg table. */
4
+ var IcebergSchemaError = class extends Data.TaggedError("IcebergSchemaError") {};
5
+ //#endregion
6
+ //#region src/iceberg/table.ts
7
+ const anonymousDeclaration = "Schemas without a name, such as Schema.Class, do not work here; use Schema.Struct";
8
+ var CompilationFailure = class extends Error {
9
+ path;
10
+ constructor(path, message) {
11
+ super(message);
12
+ this.path = path;
13
+ }
14
+ };
15
+ const fail = (path, message) => {
16
+ throw new CompilationFailure(path, message);
17
+ };
18
+ const propertyPath = (path, name) => /^[A-Za-z_$][\w$]*$/.test(name) ? `${path}.${name}` : `${path}[${JSON.stringify(name)}]`;
19
+ /** Reads annotations the way Effect does: if there are checks, the last one wins. */
20
+ const resolveAnnotations = (representation) => {
21
+ const lastCheck = representation.checks.at(-1);
22
+ return lastCheck === void 0 ? representation.annotations : lastCheck.annotations;
23
+ };
24
+ const description = (annotations) => {
25
+ const value = annotations?.description;
26
+ return typeof value === "string" ? value : void 0;
27
+ };
28
+ /** @internal */
29
+ const checkHasRepresentation = (checks, id) => checks.some((check) => check.representation?.id === id || check._tag === "FilterGroup" && checkHasRepresentation(check.checks, id));
30
+ const isAnonymousDeclaration = (representation) => representation._tag === "Declaration" && representation.representation?.id === void 0;
31
+ const literalType = (literal, path) => {
32
+ switch (typeof literal) {
33
+ case "string": return "string";
34
+ case "boolean": return "boolean";
35
+ case "number": return Number.isFinite(literal) ? "double" : fail(path, "A number literal must be a real number, not Infinity or NaN");
36
+ case "bigint": return "long";
37
+ default: return fail(path, `A ${typeof literal} literal does not work here`);
38
+ }
39
+ };
40
+ const identifierId = (fields, name) => {
41
+ const path = propertyPath("$", name);
42
+ const field = fields.find((candidate) => candidate.name === name);
43
+ if (!field) return fail(path, "Iceberg.identifier names a field that does not exist");
44
+ if (!field.required) return fail(path, "An identifier field cannot be optional or null");
45
+ if (typeof field.type !== "string" || field.type === "float" || field.type === "double") return fail(path, "An identifier field must be a primitive type other than float or double");
46
+ return field.id;
47
+ };
48
+ const assertNever = (_representation, path) => fail(path, "This kind of schema does not work here");
49
+ var Compiler = class {
50
+ references;
51
+ #activeObjects = /* @__PURE__ */ new Set();
52
+ #activeReferences = /* @__PURE__ */ new Set();
53
+ #activeSuspends = /* @__PURE__ */ new Set();
54
+ #nextId = 0;
55
+ constructor(references) {
56
+ this.references = references;
57
+ }
58
+ compile(root) {
59
+ return this.#withReference(root, "$", (representation) => {
60
+ if (isAnonymousDeclaration(representation)) return fail("$", anonymousDeclaration);
61
+ if (representation._tag !== "Objects") return fail("$", "The top level of a table must be a Schema.Struct");
62
+ const annotations = resolveAnnotations(representation);
63
+ const fields = this.#compileFields(representation, "$");
64
+ const identifierIds = (annotations?.icebergIdentifierFields ?? []).map((name) => identifierId(fields, name));
65
+ return {
66
+ schema: identifierIds.length === 0 ? {
67
+ type: "struct",
68
+ fields
69
+ } : {
70
+ type: "struct",
71
+ fields: fields.map((field) => ({
72
+ ...field,
73
+ required: identifierIds.includes(field.id)
74
+ })),
75
+ "identifier-field-ids": identifierIds
76
+ },
77
+ description: description(annotations)
78
+ };
79
+ });
80
+ }
81
+ #withReference(representation, path, compile) {
82
+ if (representation._tag !== "Reference") return compile(representation);
83
+ const reference = representation.$ref;
84
+ const resolved = this.references[reference];
85
+ if (resolved === void 0) return fail(path, `Cannot find the schema named ${reference}`);
86
+ if (this.#activeReferences.has(reference)) return fail(path, "Iceberg cannot store a schema that contains itself");
87
+ this.#activeReferences.add(reference);
88
+ try {
89
+ return this.#withReference(resolved, path, compile);
90
+ } finally {
91
+ this.#activeReferences.delete(reference);
92
+ }
93
+ }
94
+ #checkRecursion(representation, path) {
95
+ if (representation._tag === "Reference" && this.#activeReferences.has(representation.$ref)) fail(path, "Iceberg cannot store a schema that contains itself");
96
+ }
97
+ #compileFields(representation, path) {
98
+ if (representation.indexSignatures.length > 0) return fail(path, "Schema.Record does not work here; name the keys with Schema.Struct, or store it as a JSON string");
99
+ return representation.propertySignatures.map((property) => {
100
+ if (typeof property.name !== "string") return fail(propertyPath(path, String(property.name)), "Iceberg field names must be strings");
101
+ const fieldPath = propertyPath(path, property.name);
102
+ this.#checkRecursion(property.type, fieldPath);
103
+ const id = ++this.#nextId;
104
+ const compiled = this.#compileType(property.type, fieldPath);
105
+ const fieldDescription = description(property.annotations);
106
+ return {
107
+ id,
108
+ name: property.name,
109
+ type: compiled.type,
110
+ required: !property.isOptional && !compiled.nullable,
111
+ ...fieldDescription === void 0 ? {} : { doc: fieldDescription }
112
+ };
113
+ });
114
+ }
115
+ #compileType(representation, path) {
116
+ if (representation._tag === "Reference") return this.#withReference(representation, path, (resolved) => this.#compileType(resolved, path));
117
+ switch (representation._tag) {
118
+ case "String": return {
119
+ type: "string",
120
+ nullable: false
121
+ };
122
+ case "Boolean": return {
123
+ type: "boolean",
124
+ nullable: false
125
+ };
126
+ case "BigInt": return {
127
+ type: "long",
128
+ nullable: false
129
+ };
130
+ case "Number": return checkHasRepresentation(representation.checks, "effect/schema/isFinite") || checkHasRepresentation(representation.checks, "effect/schema/isInt") ? {
131
+ type: "double",
132
+ nullable: false
133
+ } : fail(path, "Plain Schema.Number does not work here; use Schema.Finite or Schema.Int");
134
+ case "Literal": return {
135
+ type: literalType(representation.literal, path),
136
+ nullable: false
137
+ };
138
+ case "Objects":
139
+ if (this.#activeObjects.has(representation)) return fail(path, "Iceberg cannot store a schema that contains itself");
140
+ this.#activeObjects.add(representation);
141
+ try {
142
+ return {
143
+ type: {
144
+ type: "struct",
145
+ fields: this.#compileFields(representation, path)
146
+ },
147
+ nullable: false
148
+ };
149
+ } finally {
150
+ this.#activeObjects.delete(representation);
151
+ }
152
+ case "Arrays": return this.#compileArray(representation, path);
153
+ case "Declaration": return this.#compileDeclaration(representation, path);
154
+ case "Union": return this.#compileUnion(representation, path);
155
+ case "Suspend":
156
+ if (this.#activeSuspends.has(representation)) return fail(path, "Iceberg cannot store a schema that contains itself");
157
+ this.#activeSuspends.add(representation);
158
+ try {
159
+ return this.#compileType(representation.thunk, path);
160
+ } finally {
161
+ this.#activeSuspends.delete(representation);
162
+ }
163
+ case "Null":
164
+ case "Undefined": return fail(path, "A field cannot contain only null or undefined");
165
+ case "Any":
166
+ case "Unknown":
167
+ case "ObjectKeyword":
168
+ case "Symbol":
169
+ case "UniqueSymbol":
170
+ case "Void":
171
+ case "Never":
172
+ case "Enum":
173
+ case "TemplateLiteral": return fail(path, `A ${representation._tag} schema does not work here`);
174
+ default: return assertNever(representation, path);
175
+ }
176
+ }
177
+ #compileArray(representation, path) {
178
+ if (representation.elements.length > 0 || representation.rest.length !== 1) return fail(path, "A tuple does not work here; use a Schema.Array where every item has the same type");
179
+ const elementPath = `${path}.element`;
180
+ const elementRepresentation = representation.rest[0];
181
+ const elementId = ++this.#nextId;
182
+ const compiled = this.#compileType(elementRepresentation, elementPath);
183
+ return {
184
+ type: {
185
+ type: "list",
186
+ "element-id": elementId,
187
+ element: compiled.type,
188
+ "element-required": !compiled.nullable
189
+ },
190
+ nullable: false
191
+ };
192
+ }
193
+ #compileDeclaration(representation, path) {
194
+ const id = representation.representation?.id;
195
+ switch (id) {
196
+ case "effect/schema/Date": return {
197
+ type: "timestamptz",
198
+ nullable: false
199
+ };
200
+ case "effect/schema/Uint8Array": return {
201
+ type: "binary",
202
+ nullable: false
203
+ };
204
+ case "effect/schema/ReadonlyMap": return this.#compileMap(representation, path);
205
+ case void 0: return fail(path, anonymousDeclaration);
206
+ default: return fail(path, `The ${id} schema does not work here`);
207
+ }
208
+ }
209
+ #compileMap(representation, path) {
210
+ if (representation.typeParameters.length !== 2) return fail(path, "ReadonlyMap needs one key schema and one value schema");
211
+ const keyPath = `${path}.key`;
212
+ const valuePath = `${path}.value`;
213
+ const keyRepresentation = representation.typeParameters[0];
214
+ const valueRepresentation = representation.typeParameters[1];
215
+ const keyId = ++this.#nextId;
216
+ const compiledKey = this.#compileType(keyRepresentation, keyPath);
217
+ if (compiledKey.nullable) return fail(keyPath, "Map keys cannot be null");
218
+ const valueId = ++this.#nextId;
219
+ const compiledValue = this.#compileType(valueRepresentation, valuePath);
220
+ return {
221
+ type: {
222
+ type: "map",
223
+ "key-id": keyId,
224
+ key: compiledKey.type,
225
+ "value-id": valueId,
226
+ value: compiledValue.type,
227
+ "value-required": !compiledValue.nullable
228
+ },
229
+ nullable: false
230
+ };
231
+ }
232
+ #compileUnion(representation, path) {
233
+ let nullable = false;
234
+ const concrete = [];
235
+ for (const member of representation.types) if (member._tag === "Null" || member._tag === "Undefined") nullable = true;
236
+ else concrete.push(member);
237
+ if (concrete.length === 0) return fail(path, "A field cannot contain only null or undefined");
238
+ if (concrete.length === 1) {
239
+ const compiled = this.#compileType(concrete[0], path);
240
+ return {
241
+ type: compiled.type,
242
+ nullable: nullable || compiled.nullable
243
+ };
244
+ }
245
+ if (!concrete.every((member) => member._tag === "Literal")) return fail(path, "A union only works when every member is a literal");
246
+ const first = literalType(concrete[0].literal, path);
247
+ for (const member of concrete.slice(1)) if (literalType(member.literal, path) !== first) return fail(path, "Every literal in a union must be the same type");
248
+ return {
249
+ type: first,
250
+ nullable
251
+ };
252
+ }
253
+ };
254
+ const compileTable = (schema) => Effect.try({
255
+ try: () => {
256
+ const document = Schema.toRepresentation(Schema.toType(schema));
257
+ return new Compiler(document.references).compile(document.representation);
258
+ },
259
+ catch: (cause) => cause instanceof CompilationFailure ? new IcebergSchemaError({
260
+ path: cause.path,
261
+ message: cause.message,
262
+ cause
263
+ }) : new IcebergSchemaError({
264
+ path: "$",
265
+ message: cause instanceof Error ? cause.message : String(cause),
266
+ cause
267
+ })
268
+ });
269
+ const makeCreateRequest = (table, options) => ({
270
+ name: options.name,
271
+ schema: table.schema,
272
+ ...options.location === void 0 ? {} : { location: options.location },
273
+ properties: {
274
+ ...table.description === void 0 ? {} : { comment: table.description },
275
+ ...options.properties,
276
+ "format-version": "3"
277
+ }
278
+ });
279
+ /**
280
+ * Builds a REST create-table request from an Effect schema.
281
+ * The catalog assigns the stored field IDs.
282
+ *
283
+ * A root `description` becomes the table `comment`, and a field `description`
284
+ * becomes its `doc`. `title` and `examples` are dropped.
285
+ *
286
+ * `Iceberg.identifier` sets the `identifier-field-ids`, and makes every other
287
+ * top-level column nullable.
288
+ *
289
+ * @example
290
+ * ```ts
291
+ * const Row = Schema.Struct({
292
+ * id: Schema.String.pipe(Schema.annotateKey({ description: "Row identifier" })),
293
+ * })
294
+ * .annotate({ description: "Connector rows" })
295
+ * .pipe(Iceberg.identifier("id"))
296
+ *
297
+ * const request = yield* Iceberg.makeCreateTableRequest(Row, {
298
+ * name: "rows",
299
+ * location: "s3://warehouse/rows"
300
+ * })
301
+ * ```
302
+ */
303
+ const makeCreateTableRequest = (schema, options) => Effect.map(compileTable(schema), (table) => makeCreateRequest(table, options));
304
+ //#endregion
305
+ export { IcebergSchemaError as a, makeCreateTableRequest as i, compileTable as n, makeCreateRequest as r, checkHasRepresentation as t };
306
+
307
+ //# sourceMappingURL=table-CxM09xf7.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"table-CxM09xf7.mjs","names":[],"sources":["../src/iceberg/error.ts","../src/iceberg/table.ts"],"sourcesContent":["import { Data } from \"effect\";\n\n/** A schema could not be turned into an Iceberg table. */\nexport class IcebergSchemaError extends Data.TaggedError(\"IcebergSchemaError\")<{\n /** Schema path, such as `$.orders.element`. */\n readonly path: string;\n readonly message: string;\n readonly cause?: unknown;\n}> {}\n","import type {\n CreateTableRequest,\n IcebergType,\n MapType,\n StructField,\n StructType,\n TableSchema,\n} from \"iceberg-js\";\n\nimport { Effect, Schema, type SchemaRepresentation } from \"effect\";\n\nimport { IcebergSchemaError } from \"./error\";\n\n/** Options for creating a table. */\nexport type CreateTableOptions = {\n readonly name: string;\n /** Where the table's files go. The catalog may choose this when omitted. */\n readonly location?: string | undefined;\n /** Extra table properties. These win over the ones from the schema. */\n readonly properties?: Readonly<Record<string, string>> | undefined;\n};\n\ntype Representation = SchemaRepresentation.Representation;\ntype Resolved = Exclude<Representation, SchemaRepresentation.Reference>;\ntype Annotations = Schema.Annotations.Annotations;\n\ntype CompiledType = {\n readonly type: IcebergType;\n readonly nullable: boolean;\n};\n\nexport type CompiledTable = {\n readonly schema: TableSchema;\n readonly description: string | undefined;\n};\n\nconst anonymousDeclaration =\n \"Schemas without a name, such as Schema.Class, do not work here; use Schema.Struct\";\n\nclass CompilationFailure extends Error {\n readonly path: string;\n\n constructor(path: string, message: string) {\n super(message);\n this.path = path;\n }\n}\n\nconst fail = (path: string, message: string): never => {\n throw new CompilationFailure(path, message);\n};\n\nconst propertyPath = (path: string, name: string): string =>\n /^[A-Za-z_$][\\w$]*$/.test(name) ? `${path}.${name}` : `${path}[${JSON.stringify(name)}]`;\n\n/** Reads annotations the way Effect does: if there are checks, the last one wins. */\nconst resolveAnnotations = (representation: Resolved): Annotations | undefined => {\n const lastCheck = representation.checks.at(-1);\n return lastCheck === undefined ? representation.annotations : lastCheck.annotations;\n};\n\nconst description = (annotations: Annotations | undefined): string | undefined => {\n const value = annotations?.description;\n return typeof value === \"string\" ? value : undefined;\n};\n\n/** @internal */\nexport const checkHasRepresentation = (\n checks: ReadonlyArray<SchemaRepresentation.Check>,\n id: string,\n): boolean =>\n checks.some(\n (check) =>\n check.representation?.id === id ||\n (check._tag === \"FilterGroup\" && checkHasRepresentation(check.checks, id)),\n );\n\nconst isAnonymousDeclaration = (representation: Resolved): boolean =>\n representation._tag === \"Declaration\" && representation.representation?.id === undefined;\n\nconst literalType = (\n literal: SchemaRepresentation.Literal[\"literal\"],\n path: string,\n): IcebergType => {\n switch (typeof literal) {\n case \"string\":\n return \"string\";\n case \"boolean\":\n return \"boolean\";\n case \"number\":\n return Number.isFinite(literal)\n ? \"double\"\n : fail(path, \"A number literal must be a real number, not Infinity or NaN\");\n case \"bigint\":\n return \"long\";\n default:\n return fail(path, `A ${typeof literal} literal does not work here`);\n }\n};\n\nconst identifierId = (fields: ReadonlyArray<StructField>, name: string): number => {\n const path = propertyPath(\"$\", name);\n const field = fields.find((candidate) => candidate.name === name);\n\n if (!field) return fail(path, \"Iceberg.identifier names a field that does not exist\");\n if (!field.required) return fail(path, \"An identifier field cannot be optional or null\");\n if (typeof field.type !== \"string\" || field.type === \"float\" || field.type === \"double\") {\n return fail(path, \"An identifier field must be a primitive type other than float or double\");\n }\n return field.id;\n};\n\nconst assertNever = (_representation: never, path: string): never =>\n fail(path, \"This kind of schema does not work here\");\n\nclass Compiler {\n readonly #activeObjects = new Set<SchemaRepresentation.Objects>();\n readonly #activeReferences = new Set<string>();\n readonly #activeSuspends = new Set<SchemaRepresentation.Suspend>();\n // The REST request needs IDs. Only the catalog's returned IDs are authoritative.\n #nextId = 0;\n\n constructor(private readonly references: SchemaRepresentation.References) {}\n\n compile(root: Representation): CompiledTable {\n return this.#withReference(root, \"$\", (representation) => {\n if (isAnonymousDeclaration(representation)) return fail(\"$\", anonymousDeclaration);\n if (representation._tag !== \"Objects\") {\n return fail(\"$\", \"The top level of a table must be a Schema.Struct\");\n }\n\n const annotations = resolveAnnotations(representation);\n const fields = this.#compileFields(representation, \"$\");\n const identifierIds = (annotations?.icebergIdentifierFields ?? []).map((name) =>\n identifierId(fields, name),\n );\n\n return {\n schema:\n identifierIds.length === 0\n ? { type: \"struct\", fields }\n : {\n type: \"struct\",\n // Deletes only send the key and version, so other columns must be nullable.\n fields: fields.map((field) => ({\n ...field,\n required: identifierIds.includes(field.id),\n })),\n \"identifier-field-ids\": identifierIds,\n },\n description: description(annotations),\n };\n });\n }\n\n #withReference<A>(\n representation: Representation,\n path: string,\n compile: (resolved: Resolved) => A,\n ): A {\n if (representation._tag !== \"Reference\") return compile(representation);\n\n const reference = representation.$ref;\n const resolved = this.references[reference];\n if (resolved === undefined) return fail(path, `Cannot find the schema named ${reference}`);\n if (this.#activeReferences.has(reference)) {\n return fail(path, \"Iceberg cannot store a schema that contains itself\");\n }\n\n this.#activeReferences.add(reference);\n try {\n return this.#withReference(resolved, path, compile);\n } finally {\n this.#activeReferences.delete(reference);\n }\n }\n\n #checkRecursion(representation: Representation, path: string): void {\n if (representation._tag === \"Reference\" && this.#activeReferences.has(representation.$ref)) {\n fail(path, \"Iceberg cannot store a schema that contains itself\");\n }\n }\n\n #compileFields(representation: SchemaRepresentation.Objects, path: string): Array<StructField> {\n if (representation.indexSignatures.length > 0) {\n return fail(\n path,\n \"Schema.Record does not work here; name the keys with Schema.Struct, or store it as a JSON string\",\n );\n }\n\n return representation.propertySignatures.map((property) => {\n if (typeof property.name !== \"string\") {\n return fail(\n propertyPath(path, String(property.name)),\n \"Iceberg field names must be strings\",\n );\n }\n\n const fieldPath = propertyPath(path, property.name);\n this.#checkRecursion(property.type, fieldPath);\n const id = ++this.#nextId;\n const compiled = this.#compileType(property.type, fieldPath);\n const fieldDescription = description(property.annotations);\n\n return {\n id,\n name: property.name,\n type: compiled.type,\n required: !property.isOptional && !compiled.nullable,\n ...(fieldDescription === undefined ? {} : { doc: fieldDescription }),\n };\n });\n }\n\n #compileType(representation: Representation, path: string): CompiledType {\n if (representation._tag === \"Reference\") {\n return this.#withReference(representation, path, (resolved) =>\n this.#compileType(resolved, path),\n );\n }\n\n switch (representation._tag) {\n case \"String\":\n return { type: \"string\", nullable: false };\n case \"Boolean\":\n return { type: \"boolean\", nullable: false };\n case \"BigInt\":\n return { type: \"long\", nullable: false };\n case \"Number\":\n // Schema.Int can exceed Iceberg int. A double holds every safe integer\n // exactly; a long would need bigint rows.\n return checkHasRepresentation(representation.checks, \"effect/schema/isFinite\") ||\n checkHasRepresentation(representation.checks, \"effect/schema/isInt\")\n ? { type: \"double\", nullable: false }\n : fail(path, \"Plain Schema.Number does not work here; use Schema.Finite or Schema.Int\");\n case \"Literal\":\n return { type: literalType(representation.literal, path), nullable: false };\n case \"Objects\": {\n if (this.#activeObjects.has(representation)) {\n return fail(path, \"Iceberg cannot store a schema that contains itself\");\n }\n\n this.#activeObjects.add(representation);\n try {\n return {\n type: {\n type: \"struct\",\n fields: this.#compileFields(representation, path),\n } satisfies StructType,\n nullable: false,\n };\n } finally {\n this.#activeObjects.delete(representation);\n }\n }\n case \"Arrays\":\n return this.#compileArray(representation, path);\n case \"Declaration\":\n return this.#compileDeclaration(representation, path);\n case \"Union\":\n return this.#compileUnion(representation, path);\n case \"Suspend\": {\n if (this.#activeSuspends.has(representation)) {\n return fail(path, \"Iceberg cannot store a schema that contains itself\");\n }\n\n this.#activeSuspends.add(representation);\n try {\n return this.#compileType(representation.thunk, path);\n } finally {\n this.#activeSuspends.delete(representation);\n }\n }\n case \"Null\":\n case \"Undefined\":\n return fail(path, \"A field cannot contain only null or undefined\");\n case \"Any\":\n case \"Unknown\":\n case \"ObjectKeyword\":\n case \"Symbol\":\n case \"UniqueSymbol\":\n case \"Void\":\n case \"Never\":\n case \"Enum\":\n case \"TemplateLiteral\":\n return fail(path, `A ${representation._tag} schema does not work here`);\n default:\n return assertNever(representation, path);\n }\n }\n\n #compileArray(representation: SchemaRepresentation.Arrays, path: string): CompiledType {\n if (representation.elements.length > 0 || representation.rest.length !== 1) {\n return fail(\n path,\n \"A tuple does not work here; use a Schema.Array where every item has the same type\",\n );\n }\n\n const elementPath = `${path}.element`;\n const elementRepresentation = representation.rest[0]!;\n const elementId = ++this.#nextId;\n const compiled = this.#compileType(elementRepresentation, elementPath);\n\n return {\n type: {\n type: \"list\",\n \"element-id\": elementId,\n element: compiled.type,\n \"element-required\": !compiled.nullable,\n },\n nullable: false,\n };\n }\n\n #compileDeclaration(\n representation: SchemaRepresentation.Declaration,\n path: string,\n ): CompiledType {\n const id = representation.representation?.id;\n\n switch (id) {\n case \"effect/schema/Date\":\n return { type: \"timestamptz\", nullable: false };\n case \"effect/schema/Uint8Array\":\n return { type: \"binary\", nullable: false };\n case \"effect/schema/ReadonlyMap\":\n return this.#compileMap(representation, path);\n case undefined:\n return fail(path, anonymousDeclaration);\n default:\n return fail(path, `The ${id} schema does not work here`);\n }\n }\n\n #compileMap(representation: SchemaRepresentation.Declaration, path: string): CompiledType {\n if (representation.typeParameters.length !== 2) {\n return fail(path, \"ReadonlyMap needs one key schema and one value schema\");\n }\n\n const keyPath = `${path}.key`;\n const valuePath = `${path}.value`;\n const keyRepresentation = representation.typeParameters[0]!;\n const valueRepresentation = representation.typeParameters[1]!;\n const keyId = ++this.#nextId;\n const compiledKey = this.#compileType(keyRepresentation, keyPath);\n\n if (compiledKey.nullable) return fail(keyPath, \"Map keys cannot be null\");\n\n const valueId = ++this.#nextId;\n const compiledValue = this.#compileType(valueRepresentation, valuePath);\n\n return {\n type: {\n type: \"map\",\n \"key-id\": keyId,\n key: compiledKey.type,\n \"value-id\": valueId,\n value: compiledValue.type,\n \"value-required\": !compiledValue.nullable,\n } satisfies MapType,\n nullable: false,\n };\n }\n\n #compileUnion(representation: SchemaRepresentation.Union, path: string): CompiledType {\n let nullable = false;\n const concrete: Array<Representation> = [];\n\n for (const member of representation.types) {\n if (member._tag === \"Null\" || member._tag === \"Undefined\") {\n nullable = true;\n } else {\n concrete.push(member);\n }\n }\n\n if (concrete.length === 0) return fail(path, \"A field cannot contain only null or undefined\");\n\n if (concrete.length === 1) {\n const compiled = this.#compileType(concrete[0]!, path);\n return { type: compiled.type, nullable: nullable || compiled.nullable };\n }\n\n if (!concrete.every((member) => member._tag === \"Literal\")) {\n return fail(path, \"A union only works when every member is a literal\");\n }\n\n const first = literalType(concrete[0]!.literal, path);\n for (const member of concrete.slice(1)) {\n if (literalType(member.literal, path) !== first) {\n return fail(path, \"Every literal in a union must be the same type\");\n }\n }\n\n return { type: first, nullable };\n }\n}\n\nexport const compileTable = (\n schema: Schema.Top,\n): Effect.Effect<CompiledTable, IcebergSchemaError> =>\n Effect.try({\n try: () => {\n const document = Schema.toRepresentation(Schema.toType(schema));\n return new Compiler(document.references).compile(document.representation);\n },\n catch: (cause) =>\n cause instanceof CompilationFailure\n ? new IcebergSchemaError({ path: cause.path, message: cause.message, cause })\n : new IcebergSchemaError({\n path: \"$\",\n message: cause instanceof Error ? cause.message : String(cause),\n cause,\n }),\n });\n\nexport const makeCreateRequest = (\n table: CompiledTable,\n options: CreateTableOptions,\n): CreateTableRequest => ({\n name: options.name,\n schema: table.schema,\n ...(options.location === undefined ? {} : { location: options.location }),\n properties: {\n ...(table.description === undefined ? {} : { comment: table.description }),\n ...options.properties,\n \"format-version\": \"3\",\n },\n});\n\n/**\n * Builds a REST create-table request from an Effect schema.\n * The catalog assigns the stored field IDs.\n *\n * A root `description` becomes the table `comment`, and a field `description`\n * becomes its `doc`. `title` and `examples` are dropped.\n *\n * `Iceberg.identifier` sets the `identifier-field-ids`, and makes every other\n * top-level column nullable.\n *\n * @example\n * ```ts\n * const Row = Schema.Struct({\n * id: Schema.String.pipe(Schema.annotateKey({ description: \"Row identifier\" })),\n * })\n * .annotate({ description: \"Connector rows\" })\n * .pipe(Iceberg.identifier(\"id\"))\n *\n * const request = yield* Iceberg.makeCreateTableRequest(Row, {\n * name: \"rows\",\n * location: \"s3://warehouse/rows\"\n * })\n * ```\n */\nexport const makeCreateTableRequest = (\n schema: Schema.Top,\n options: CreateTableOptions,\n): Effect.Effect<CreateTableRequest, IcebergSchemaError> =>\n Effect.map(compileTable(schema), (table) => makeCreateRequest(table, options));\n"],"mappings":";;;AAGA,IAAa,qBAAb,cAAwC,KAAK,YAAY,oBAAoB,CAAC,CAK3E,CAAC;;;AC4BJ,MAAM,uBACJ;AAEF,IAAM,qBAAN,cAAiC,MAAM;CACrC;CAEA,YAAY,MAAc,SAAiB;EACzC,MAAM,OAAO;EACb,KAAK,OAAO;CACd;AACF;AAEA,MAAM,QAAQ,MAAc,YAA2B;CACrD,MAAM,IAAI,mBAAmB,MAAM,OAAO;AAC5C;AAEA,MAAM,gBAAgB,MAAc,SAClC,qBAAqB,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,GAAG,KAAK,UAAU,IAAI,EAAE;;AAGxF,MAAM,sBAAsB,mBAAsD;CAChF,MAAM,YAAY,eAAe,OAAO,GAAG,EAAE;CAC7C,OAAO,cAAc,KAAA,IAAY,eAAe,cAAc,UAAU;AAC1E;AAEA,MAAM,eAAe,gBAA6D;CAChF,MAAM,QAAQ,aAAa;CAC3B,OAAO,OAAO,UAAU,WAAW,QAAQ,KAAA;AAC7C;;AAGA,MAAa,0BACX,QACA,OAEA,OAAO,MACJ,UACC,MAAM,gBAAgB,OAAO,MAC5B,MAAM,SAAS,iBAAiB,uBAAuB,MAAM,QAAQ,EAAE,CAC5E;AAEF,MAAM,0BAA0B,mBAC9B,eAAe,SAAS,iBAAiB,eAAe,gBAAgB,OAAO,KAAA;AAEjF,MAAM,eACJ,SACA,SACgB;CAChB,QAAQ,OAAO,SAAf;EACE,KAAK,UACH,OAAO;EACT,KAAK,WACH,OAAO;EACT,KAAK,UACH,OAAO,OAAO,SAAS,OAAO,IAC1B,WACA,KAAK,MAAM,6DAA6D;EAC9E,KAAK,UACH,OAAO;EACT,SACE,OAAO,KAAK,MAAM,KAAK,OAAO,QAAQ,4BAA4B;CACtE;AACF;AAEA,MAAM,gBAAgB,QAAoC,SAAyB;CACjF,MAAM,OAAO,aAAa,KAAK,IAAI;CACnC,MAAM,QAAQ,OAAO,MAAM,cAAc,UAAU,SAAS,IAAI;CAEhE,IAAI,CAAC,OAAO,OAAO,KAAK,MAAM,sDAAsD;CACpF,IAAI,CAAC,MAAM,UAAU,OAAO,KAAK,MAAM,gDAAgD;CACvF,IAAI,OAAO,MAAM,SAAS,YAAY,MAAM,SAAS,WAAW,MAAM,SAAS,UAC7E,OAAO,KAAK,MAAM,yEAAyE;CAE7F,OAAO,MAAM;AACf;AAEA,MAAM,eAAe,iBAAwB,SAC3C,KAAK,MAAM,wCAAwC;AAErD,IAAM,WAAN,MAAe;CAOgB;CAN7B,iCAA0B,IAAI,IAAkC;CAChE,oCAA6B,IAAI,IAAY;CAC7C,kCAA2B,IAAI,IAAkC;CAEjE,UAAU;CAEV,YAAY,YAA8D;EAA7C,KAAA,aAAA;CAA8C;CAE3E,QAAQ,MAAqC;EAC3C,OAAO,KAAK,eAAe,MAAM,MAAM,mBAAmB;GACxD,IAAI,uBAAuB,cAAc,GAAG,OAAO,KAAK,KAAK,oBAAoB;GACjF,IAAI,eAAe,SAAS,WAC1B,OAAO,KAAK,KAAK,kDAAkD;GAGrE,MAAM,cAAc,mBAAmB,cAAc;GACrD,MAAM,SAAS,KAAK,eAAe,gBAAgB,GAAG;GACtD,MAAM,iBAAiB,aAAa,2BAA2B,CAAC,EAAA,CAAG,KAAK,SACtE,aAAa,QAAQ,IAAI,CAC3B;GAEA,OAAO;IACL,QACE,cAAc,WAAW,IACrB;KAAE,MAAM;KAAU;IAAO,IACzB;KACE,MAAM;KAEN,QAAQ,OAAO,KAAK,WAAW;MAC7B,GAAG;MACH,UAAU,cAAc,SAAS,MAAM,EAAE;KAC3C,EAAE;KACF,wBAAwB;IAC1B;IACN,aAAa,YAAY,WAAW;GACtC;EACF,CAAC;CACH;CAEA,eACE,gBACA,MACA,SACG;EACH,IAAI,eAAe,SAAS,aAAa,OAAO,QAAQ,cAAc;EAEtE,MAAM,YAAY,eAAe;EACjC,MAAM,WAAW,KAAK,WAAW;EACjC,IAAI,aAAa,KAAA,GAAW,OAAO,KAAK,MAAM,gCAAgC,WAAW;EACzF,IAAI,KAAK,kBAAkB,IAAI,SAAS,GACtC,OAAO,KAAK,MAAM,oDAAoD;EAGxE,KAAK,kBAAkB,IAAI,SAAS;EACpC,IAAI;GACF,OAAO,KAAK,eAAe,UAAU,MAAM,OAAO;EACpD,UAAU;GACR,KAAK,kBAAkB,OAAO,SAAS;EACzC;CACF;CAEA,gBAAgB,gBAAgC,MAAoB;EAClE,IAAI,eAAe,SAAS,eAAe,KAAK,kBAAkB,IAAI,eAAe,IAAI,GACvF,KAAK,MAAM,oDAAoD;CAEnE;CAEA,eAAe,gBAA8C,MAAkC;EAC7F,IAAI,eAAe,gBAAgB,SAAS,GAC1C,OAAO,KACL,MACA,kGACF;EAGF,OAAO,eAAe,mBAAmB,KAAK,aAAa;GACzD,IAAI,OAAO,SAAS,SAAS,UAC3B,OAAO,KACL,aAAa,MAAM,OAAO,SAAS,IAAI,CAAC,GACxC,qCACF;GAGF,MAAM,YAAY,aAAa,MAAM,SAAS,IAAI;GAClD,KAAK,gBAAgB,SAAS,MAAM,SAAS;GAC7C,MAAM,KAAK,EAAE,KAAK;GAClB,MAAM,WAAW,KAAK,aAAa,SAAS,MAAM,SAAS;GAC3D,MAAM,mBAAmB,YAAY,SAAS,WAAW;GAEzD,OAAO;IACL;IACA,MAAM,SAAS;IACf,MAAM,SAAS;IACf,UAAU,CAAC,SAAS,cAAc,CAAC,SAAS;IAC5C,GAAI,qBAAqB,KAAA,IAAY,CAAC,IAAI,EAAE,KAAK,iBAAiB;GACpE;EACF,CAAC;CACH;CAEA,aAAa,gBAAgC,MAA4B;EACvE,IAAI,eAAe,SAAS,aAC1B,OAAO,KAAK,eAAe,gBAAgB,OAAO,aAChD,KAAK,aAAa,UAAU,IAAI,CAClC;EAGF,QAAQ,eAAe,MAAvB;GACE,KAAK,UACH,OAAO;IAAE,MAAM;IAAU,UAAU;GAAM;GAC3C,KAAK,WACH,OAAO;IAAE,MAAM;IAAW,UAAU;GAAM;GAC5C,KAAK,UACH,OAAO;IAAE,MAAM;IAAQ,UAAU;GAAM;GACzC,KAAK,UAGH,OAAO,uBAAuB,eAAe,QAAQ,wBAAwB,KAC3E,uBAAuB,eAAe,QAAQ,qBAAqB,IACjE;IAAE,MAAM;IAAU,UAAU;GAAM,IAClC,KAAK,MAAM,yEAAyE;GAC1F,KAAK,WACH,OAAO;IAAE,MAAM,YAAY,eAAe,SAAS,IAAI;IAAG,UAAU;GAAM;GAC5E,KAAK;IACH,IAAI,KAAK,eAAe,IAAI,cAAc,GACxC,OAAO,KAAK,MAAM,oDAAoD;IAGxE,KAAK,eAAe,IAAI,cAAc;IACtC,IAAI;KACF,OAAO;MACL,MAAM;OACJ,MAAM;OACN,QAAQ,KAAK,eAAe,gBAAgB,IAAI;MAClD;MACA,UAAU;KACZ;IACF,UAAU;KACR,KAAK,eAAe,OAAO,cAAc;IAC3C;GAEF,KAAK,UACH,OAAO,KAAK,cAAc,gBAAgB,IAAI;GAChD,KAAK,eACH,OAAO,KAAK,oBAAoB,gBAAgB,IAAI;GACtD,KAAK,SACH,OAAO,KAAK,cAAc,gBAAgB,IAAI;GAChD,KAAK;IACH,IAAI,KAAK,gBAAgB,IAAI,cAAc,GACzC,OAAO,KAAK,MAAM,oDAAoD;IAGxE,KAAK,gBAAgB,IAAI,cAAc;IACvC,IAAI;KACF,OAAO,KAAK,aAAa,eAAe,OAAO,IAAI;IACrD,UAAU;KACR,KAAK,gBAAgB,OAAO,cAAc;IAC5C;GAEF,KAAK;GACL,KAAK,aACH,OAAO,KAAK,MAAM,+CAA+C;GACnE,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK;GACL,KAAK,mBACH,OAAO,KAAK,MAAM,KAAK,eAAe,KAAK,2BAA2B;GACxE,SACE,OAAO,YAAY,gBAAgB,IAAI;EAC3C;CACF;CAEA,cAAc,gBAA6C,MAA4B;EACrF,IAAI,eAAe,SAAS,SAAS,KAAK,eAAe,KAAK,WAAW,GACvE,OAAO,KACL,MACA,mFACF;EAGF,MAAM,cAAc,GAAG,KAAK;EAC5B,MAAM,wBAAwB,eAAe,KAAK;EAClD,MAAM,YAAY,EAAE,KAAK;EACzB,MAAM,WAAW,KAAK,aAAa,uBAAuB,WAAW;EAErE,OAAO;GACL,MAAM;IACJ,MAAM;IACN,cAAc;IACd,SAAS,SAAS;IAClB,oBAAoB,CAAC,SAAS;GAChC;GACA,UAAU;EACZ;CACF;CAEA,oBACE,gBACA,MACc;EACd,MAAM,KAAK,eAAe,gBAAgB;EAE1C,QAAQ,IAAR;GACE,KAAK,sBACH,OAAO;IAAE,MAAM;IAAe,UAAU;GAAM;GAChD,KAAK,4BACH,OAAO;IAAE,MAAM;IAAU,UAAU;GAAM;GAC3C,KAAK,6BACH,OAAO,KAAK,YAAY,gBAAgB,IAAI;GAC9C,KAAK,KAAA,GACH,OAAO,KAAK,MAAM,oBAAoB;GACxC,SACE,OAAO,KAAK,MAAM,OAAO,GAAG,2BAA2B;EAC3D;CACF;CAEA,YAAY,gBAAkD,MAA4B;EACxF,IAAI,eAAe,eAAe,WAAW,GAC3C,OAAO,KAAK,MAAM,uDAAuD;EAG3E,MAAM,UAAU,GAAG,KAAK;EACxB,MAAM,YAAY,GAAG,KAAK;EAC1B,MAAM,oBAAoB,eAAe,eAAe;EACxD,MAAM,sBAAsB,eAAe,eAAe;EAC1D,MAAM,QAAQ,EAAE,KAAK;EACrB,MAAM,cAAc,KAAK,aAAa,mBAAmB,OAAO;EAEhE,IAAI,YAAY,UAAU,OAAO,KAAK,SAAS,yBAAyB;EAExE,MAAM,UAAU,EAAE,KAAK;EACvB,MAAM,gBAAgB,KAAK,aAAa,qBAAqB,SAAS;EAEtE,OAAO;GACL,MAAM;IACJ,MAAM;IACN,UAAU;IACV,KAAK,YAAY;IACjB,YAAY;IACZ,OAAO,cAAc;IACrB,kBAAkB,CAAC,cAAc;GACnC;GACA,UAAU;EACZ;CACF;CAEA,cAAc,gBAA4C,MAA4B;EACpF,IAAI,WAAW;EACf,MAAM,WAAkC,CAAC;EAEzC,KAAK,MAAM,UAAU,eAAe,OAClC,IAAI,OAAO,SAAS,UAAU,OAAO,SAAS,aAC5C,WAAW;OAEX,SAAS,KAAK,MAAM;EAIxB,IAAI,SAAS,WAAW,GAAG,OAAO,KAAK,MAAM,+CAA+C;EAE5F,IAAI,SAAS,WAAW,GAAG;GACzB,MAAM,WAAW,KAAK,aAAa,SAAS,IAAK,IAAI;GACrD,OAAO;IAAE,MAAM,SAAS;IAAM,UAAU,YAAY,SAAS;GAAS;EACxE;EAEA,IAAI,CAAC,SAAS,OAAO,WAAW,OAAO,SAAS,SAAS,GACvD,OAAO,KAAK,MAAM,mDAAmD;EAGvE,MAAM,QAAQ,YAAY,SAAS,EAAE,CAAE,SAAS,IAAI;EACpD,KAAK,MAAM,UAAU,SAAS,MAAM,CAAC,GACnC,IAAI,YAAY,OAAO,SAAS,IAAI,MAAM,OACxC,OAAO,KAAK,MAAM,gDAAgD;EAItE,OAAO;GAAE,MAAM;GAAO;EAAS;CACjC;AACF;AAEA,MAAa,gBACX,WAEA,OAAO,IAAI;CACT,WAAW;EACT,MAAM,WAAW,OAAO,iBAAiB,OAAO,OAAO,MAAM,CAAC;EAC9D,OAAO,IAAI,SAAS,SAAS,UAAU,CAAC,CAAC,QAAQ,SAAS,cAAc;CAC1E;CACA,QAAQ,UACN,iBAAiB,qBACb,IAAI,mBAAmB;EAAE,MAAM,MAAM;EAAM,SAAS,MAAM;EAAS;CAAM,CAAC,IAC1E,IAAI,mBAAmB;EACrB,MAAM;EACN,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC9D;CACF,CAAC;AACT,CAAC;AAEH,MAAa,qBACX,OACA,aACwB;CACxB,MAAM,QAAQ;CACd,QAAQ,MAAM;CACd,GAAI,QAAQ,aAAa,KAAA,IAAY,CAAC,IAAI,EAAE,UAAU,QAAQ,SAAS;CACvE,YAAY;EACV,GAAI,MAAM,gBAAgB,KAAA,IAAY,CAAC,IAAI,EAAE,SAAS,MAAM,YAAY;EACxE,GAAG,QAAQ;EACX,kBAAkB;CACpB;AACF;;;;;;;;;;;;;;;;;;;;;;;;;AA0BA,MAAa,0BACX,QACA,YAEA,OAAO,IAAI,aAAa,MAAM,IAAI,UAAU,kBAAkB,OAAO,OAAO,CAAC"}
@@ -0,0 +1,220 @@
1
+ import { t as ConnectorError } from "./index-DkVNO6ks.mjs";
2
+ import { n as RequiredField, t as Identified } from "./annotations-vzSzXtr-.mjs";
3
+ import { Duration, Effect, Schema } from "effect";
4
+ import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/http";
5
+ //#region src/core/types.d.ts
6
+ declare namespace Cursor {
7
+ type Value = string | number | Date;
8
+ type Kind = "string" | "number" | "isoDateTime";
9
+ type Definition<T extends Value = Value> = {
10
+ readonly kind: Kind;
11
+ readonly decode: (value: unknown) => Effect.Effect<T, ConnectorError>;
12
+ };
13
+ }
14
+ /** Rows and an optional cursor from one resource. */
15
+ type ResourceBatch<Row extends object = object> = {
16
+ readonly cursor?: Cursor.Value;
17
+ readonly rows: ReadonlyArray<Row>;
18
+ };
19
+ type ResourceState = {
20
+ readonly backfill?: {
21
+ readonly cutoff: Cursor.Value;
22
+ readonly pageCursor?: Cursor.Value;
23
+ readonly completed: boolean;
24
+ readonly lastSuccessAt?: string;
25
+ };
26
+ readonly changes?: {
27
+ readonly cursor: Cursor.Value;
28
+ readonly lastSuccessAt?: string;
29
+ };
30
+ readonly lastError?: {
31
+ readonly source: "backfill" | "changes";
32
+ readonly operation: "fetch" | "ingest" | "checkpoint";
33
+ readonly code: "fetch_failed" | "ingest_failed" | "checkpoint_failed";
34
+ readonly message: string;
35
+ readonly at: string;
36
+ };
37
+ };
38
+ type SyncState = "pending" | "backfilling" | "live" | "error";
39
+ /** A row schema with its key, set by `Iceberg.identifier`. */
40
+ type ResourceSchema = Schema.Decoder<object> & Identified<string>;
41
+ /**
42
+ * Takes a resource schema and returns its decoded row object type.
43
+ *
44
+ * @example
45
+ * ```ts
46
+ * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String }).pipe(
47
+ * Iceberg.identifier("id"),
48
+ * );
49
+ * type Product = ResourceRow<typeof ProductSchema>;
50
+ * // Result:
51
+ * // type Product = { readonly id: string; readonly updatedAt: string }
52
+ * ```
53
+ */
54
+ type ResourceRow<S extends ResourceSchema> = Schema.Schema.Type<S>;
55
+ /**
56
+ * Takes a resource schema and returns the string keys available on its row type.
57
+ *
58
+ * @example
59
+ * ```ts
60
+ * const ProductSchema = Schema.Struct({ id: Schema.String, updatedAt: Schema.String }).pipe(
61
+ * Iceberg.identifier("id"),
62
+ * );
63
+ * type ProductField = ResourceField<typeof ProductSchema>;
64
+ * // Result:
65
+ * // type ProductField = "id" | "updatedAt"
66
+ *
67
+ * const version: ResourceField<typeof ProductSchema> = "updatedAt";
68
+ * ```
69
+ */
70
+ type KnownResourceField<S extends ResourceSchema> = keyof ResourceRow<S> & string;
71
+ type ResourceField<S extends ResourceSchema> = [KnownResourceField<S>] extends [never] ? string : KnownResourceField<S>;
72
+ /** A row field that cannot be null or undefined. */
73
+ type ResourceRequiredField<S extends ResourceSchema> = RequiredField<S>;
74
+ /** The fields that identify a row, from `Iceberg.identifier`. */
75
+ type ResourceKey<S extends ResourceSchema> = S extends Identified<infer Key> ? Key : never;
76
+ /** Deletes a row: its key, version, and `_deleted: true`. */
77
+ type ResourceDelete<S extends ResourceSchema, Version extends ResourceField<S>> = Pick<ResourceRow<S>, Extract<ResourceKey<S> | Version, keyof ResourceRow<S>>> & {
78
+ readonly _deleted: true;
79
+ };
80
+ /** A full row, or a delete. The engine replaces the whole row. */
81
+ type ResourceUpdate<S extends ResourceSchema, Version extends ResourceField<S>> = (ResourceRow<S> & {
82
+ readonly _deleted?: boolean;
83
+ }) | ResourceDelete<S, Version>;
84
+ type FetchPageResult<Row extends object> = {
85
+ readonly rows: ReadonlyArray<Row>;
86
+ readonly nextPageCursor?: Cursor.Value;
87
+ readonly hasMore: boolean;
88
+ };
89
+ type PageFetch<Row extends object, R = never> = {
90
+ readonly pageCursor: Cursor.Definition;
91
+ readonly cutoff: Cursor.Definition;
92
+ readonly fetch: (input: {
93
+ readonly pageCursor?: Cursor.Value;
94
+ readonly cutoff: Cursor.Value;
95
+ }) => Effect.Effect<FetchPageResult<Row>, ConnectorError, R>;
96
+ };
97
+ type FetchChangesResult<Row extends object> = {
98
+ readonly rows: ReadonlyArray<Row>;
99
+ readonly cursor: Cursor.Value;
100
+ /** When true, the engine runs the changes again without waiting for the interval. */
101
+ readonly hasMore?: boolean;
102
+ };
103
+ type ChangesFetch<Row extends object, R = never> = {
104
+ readonly cursor: Cursor.Definition;
105
+ readonly interval?: Duration.Input;
106
+ readonly fetch: (input: {
107
+ readonly cursor: Cursor.Value;
108
+ }) => Effect.Effect<FetchChangesResult<Row>, ConnectorError, R>;
109
+ };
110
+ type WebhookHandler<Row extends object, Payload> = {
111
+ /** Types the payload. The route decodes it before `to`. */
112
+ readonly schema: Schema.Decoder<Payload>;
113
+ handler(input: {
114
+ readonly payload: Payload;
115
+ }): Effect.Effect<ReadonlyArray<Row>, ConnectorError>;
116
+ };
117
+ type ResourceDefinition<S extends ResourceSchema = ResourceSchema, Payload = unknown, R = never, Name extends string = string, Version extends ResourceRequiredField<S> = ResourceRequiredField<S>> = {
118
+ readonly name: Name;
119
+ /** Types the rows, and names the key. The Iceberg table decides how they are encoded. */
120
+ readonly rowSchema: S;
121
+ /** Orders updates for the entity. */
122
+ readonly version: Version;
123
+ /** Read-only validation used before provisioning this resource. */
124
+ readonly check: Effect.Effect<void, ConnectorError>;
125
+ readonly backfill?: PageFetch<ResourceRow<NoInfer<S>>, R>;
126
+ readonly changes?: ChangesFetch<ResourceUpdate<NoInfer<S>, NoInfer<Version>>, R>;
127
+ readonly webhook?: WebhookHandler<ResourceUpdate<NoInfer<S>, NoInfer<Version>>, Payload>;
128
+ };
129
+ /** The parts of a resource its table needs. */
130
+ type TableResource = Pick<ResourceDefinition, "name" | "rowSchema" | "version">;
131
+ /** Resources that write to tables, such as a connector or a demo. */
132
+ type TableSource = {
133
+ readonly resources: ReadonlyArray<TableResource>;
134
+ };
135
+ /**
136
+ * Takes a connector resource tuple and returns a `{ [resourceName]: rowType }`
137
+ * map.
138
+ *
139
+ * @example
140
+ * ```ts
141
+ * type Rows = ResourceRows<[typeof Products, typeof Orders]>;
142
+ * // Result:
143
+ * // type Rows = {
144
+ * // readonly products: Product;
145
+ * // readonly orders: Order;
146
+ * // }
147
+ *
148
+ * type ProductRow = Rows["products"];
149
+ * // Result:
150
+ * // type ProductRow = Product
151
+ * ```
152
+ */
153
+ type ResourceRows<Resources extends ReadonlyArray<ResourceDefinition>> = { readonly [Resource in Resources[number] as Resource["name"]]: Resource extends ResourceDefinition<infer S, unknown, unknown, string, infer _Version> ? ResourceRow<S> : never; };
154
+ /** Resource names preserved from a connector's resource tuple. */
155
+ type ResourceName<Resources extends ReadonlyArray<ResourceDefinition>> = keyof ResourceRows<Resources> & string;
156
+ /**
157
+ * Takes a resource definition and returns the payload type accepted by
158
+ * `to(resource, payload)`.
159
+ *
160
+ * @example
161
+ * ```ts
162
+ * type ProductPayload = ResourcePayload<typeof Products>;
163
+ * // Result, if Products was defined with a ProductEvent webhook schema:
164
+ * // type ProductPayload = ProductEvent
165
+ * ```
166
+ */
167
+ type ResourcePayload<R> = R extends ResourceDefinition<ResourceSchema, infer Payload, unknown, string, string> ? Payload : never;
168
+ type WebhookAckMode = "after-enqueue" | "after-ingest";
169
+ type WebhookRouteContext<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, Payload = unknown> = {
170
+ readonly request: HttpServerRequest.HttpServerRequest;
171
+ readonly rawBody: Uint8Array;
172
+ readonly payload: Payload;
173
+ /** Runs a resource's webhook handler on a decoded payload and collects its rows. */
174
+ readonly to: <Resource extends Resources[number]>(resource: Resource, payload: ResourcePayload<Resource>) => Effect.Effect<void, ConnectorError>;
175
+ };
176
+ type WebhookRoute<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, Payload = unknown> = {
177
+ readonly path: HttpRouter.PathInput;
178
+ readonly ackMode: WebhookAckMode;
179
+ readonly schema: Schema.Decoder<Payload>;
180
+ handler(context: WebhookRouteContext<Resources, Payload>): Effect.Effect<HttpServerResponse.HttpServerResponse, ConnectorError>;
181
+ };
182
+ type WebhookRouteInput<Resources extends ReadonlyArray<ResourceDefinition>, Payload> = WebhookRoute<Resources, Payload>;
183
+ /**
184
+ * Takes a resource definition and returns the rows its changes and webhooks may
185
+ * return: full rows or deletes.
186
+ */
187
+ type ResourceUpdateOf<R> = R extends ResourceDefinition<infer S, unknown, unknown, string, infer Version> ? ResourceUpdate<S, Version> : never;
188
+ /** Rows from one feed run, by resource name. A missing resource has no rows. */
189
+ type FeedRows<Resources extends ReadonlyArray<ResourceDefinition>> = { readonly [Resource in Resources[number] as Resource["name"]]?: ReadonlyArray<ResourceUpdateOf<Resource>>; };
190
+ type FetchFeedResult<Resources extends ReadonlyArray<ResourceDefinition>> = {
191
+ readonly rows: FeedRows<Resources>;
192
+ readonly cursor: Cursor.Value;
193
+ /** When true, the engine runs the feed again without waiting for the interval. */
194
+ readonly hasMore?: boolean;
195
+ };
196
+ /**
197
+ * One changes source for several resources, with one cursor. Use it when one
198
+ * provider log, such as an events API, covers them all.
199
+ */
200
+ type ChangesFeed<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, R = never> = {
201
+ readonly resources: Resources;
202
+ readonly cursor: Cursor.Definition;
203
+ readonly interval?: Duration.Input;
204
+ readonly fetch: (input: {
205
+ readonly cursor: Cursor.Value;
206
+ /** The resources that run. Skip the work for the others. */
207
+ readonly selected: ReadonlySet<string>;
208
+ }) => Effect.Effect<FetchFeedResult<Resources>, ConnectorError, R>;
209
+ };
210
+ type ConnectorDefinition<Resources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>, FeedResources extends ReadonlyArray<ResourceDefinition> = ReadonlyArray<ResourceDefinition>> = {
211
+ readonly name: string;
212
+ readonly title?: string;
213
+ readonly resources: Resources;
214
+ /** Shared changes for resources that do not define their own `changes`. */
215
+ readonly changes?: ChangesFeed<FeedResources>;
216
+ readonly webhooks?: ReadonlyArray<WebhookRoute>;
217
+ };
218
+ //#endregion
219
+ export { WebhookRouteContext as A, ResourceUpdateOf as C, WebhookAckMode as D, TableSource as E, WebhookHandler as O, ResourceUpdate as S, TableResource as T, ResourceRequiredField as _, FeedRows as a, ResourceSchema as b, FetchPageResult as c, ResourceDefinition as d, ResourceDelete as f, ResourcePayload as g, ResourceName as h, Cursor as i, WebhookRouteInput as j, WebhookRoute as k, PageFetch as l, ResourceKey as m, ChangesFetch as n, FetchChangesResult as o, ResourceField as p, ConnectorDefinition as r, FetchFeedResult as s, ChangesFeed as t, ResourceBatch as u, ResourceRow as v, SyncState as w, ResourceState as x, ResourceRows as y };
220
+ //# sourceMappingURL=types-NtSv6gwl.d.mts.map
@@ -0,0 +1,3 @@
1
+ import { A as WebhookRouteContext, D as WebhookAckMode, j as WebhookRouteInput, k as WebhookRoute } from "../types-NtSv6gwl.mjs";
2
+ import { n as route, r as router } from "../index-dHS59EZl.mjs";
3
+ export { type WebhookAckMode, type WebhookRoute, type WebhookRouteContext, type WebhookRouteInput, route, router };
@@ -0,0 +1,12 @@
1
+ import { t as __exportAll } from "../rolldown-runtime-D7D4PA-g.mjs";
2
+ import { n as router } from "../server-D1TTN0_D.mjs";
3
+ //#region src/webhook/index.ts
4
+ var webhook_exports = /* @__PURE__ */ __exportAll({
5
+ route: () => route,
6
+ router: () => router
7
+ });
8
+ const route = (definition) => definition;
9
+ //#endregion
10
+ export { route, router, webhook_exports as t };
11
+
12
+ //# sourceMappingURL=index.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.mjs","names":[],"sources":["../../src/webhook/index.ts"],"sourcesContent":["import type { ResourceDefinition, WebhookRoute, WebhookRouteInput } from \"../core/types\";\n\nexport { router } from \"./server\";\nexport type { WebhookAckMode, WebhookRoute, WebhookRouteContext, WebhookRouteInput } from \"./types\";\n\nexport const route = <const Resources extends ReadonlyArray<ResourceDefinition>, Payload>(\n definition: WebhookRouteInput<Resources, Payload>,\n): WebhookRoute<Resources, Payload> => definition;\n"],"mappings":";;;;;;;AAKA,MAAa,SACX,eACqC"}