@eltjs/elt-postgresql 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  import { hash } from 'node:crypto';
2
2
  import { Lock } from '@zukhruf/async';
3
- import { DerivedDataset, errorMessage, } from '@eltjs/elt';
3
+ import { DerivedDataset, computedColumns, refreshComputed, } from '@eltjs/elt';
4
4
  import { identifier, quote } from './identifier.js';
5
5
  import { TargetComments } from './postgres-comments.js';
6
6
  import { rebindDependents } from './postgres-dependents.js';
@@ -12,16 +12,9 @@ import { PostgresLoad } from './postgres-load.js';
12
12
  import { TargetReaderView } from './postgres-reader-view.js';
13
13
  import { makeSchema } from './postgres-session.js';
14
14
  import { View, made } from './view.js';
15
- // The columns the dataset keeps on every output row beside the rule's own.
16
- const engineColumns = {
17
- input_hash: 'SHA-256 of the input the row was made from; the row is made again when its input changes.',
18
- rule: 'What made the row, such as the parser of its format or the model; the row is made again when that changes. unsupported: the rule does not handle the input.',
19
- status: 'computed: the rule made the row. empty: the rule found nothing to keep, such as a photo without words. failed: the rule failed, error says why, and the refresh after the passes tries again, at most once for each pass.',
20
- error: 'Why the rule failed. NULL unless status is failed.',
21
- };
22
15
  // The names a rule column cannot take: the dataset's own, and the history's.
23
16
  const ownColumns = [
24
- ...Object.keys(engineColumns),
17
+ ...Object.keys(computedColumns),
25
18
  'committed_in',
26
19
  '_elt_row_hash',
27
20
  'replaced_in',
@@ -74,7 +67,7 @@ export class ComputedDataset extends DerivedDataset {
74
67
  type: 'text',
75
68
  constraint: ' PRIMARY KEY',
76
69
  },
77
- ...Object.keys(engineColumns).map((column) => ({
70
+ ...Object.keys(computedColumns).map((column) => ({
78
71
  name: column,
79
72
  type: 'text',
80
73
  constraint: column === 'error' ? '' : ' NOT NULL',
@@ -91,7 +84,7 @@ export class ComputedDataset extends DerivedDataset {
91
84
  this.#history = new TargetHistory(into.schema, hash('sha256', JSON.stringify([into.schema, this.#table])).slice(0, 40));
92
85
  this.#view = new TargetReaderView({ schema: into.schema, name }, table, this.#history.qualifiedName, this.#columns, new TargetComments(description, {
93
86
  [compute.key.name]: compute.key.description,
94
- ...engineColumns,
87
+ ...computedColumns,
95
88
  ...Object.fromEntries(Object.entries(compute.output).map(([column, { description }]) => [
96
89
  column,
97
90
  description,
@@ -124,28 +117,23 @@ export class ComputedDataset extends DerivedDataset {
124
117
  // no commit it is no span, and leaves the positions as they are.
125
118
  await using load = await PostgresLoad.open(this.into.url, { dataset: this.name }, { clock, retention: this.into.retention });
126
119
  const status = new PostgresDerivedStatus(load.sql, this.qualifiedName, 'computed', clock);
127
- const inputs = await this.#inputs(load.sql, connections);
128
- if (inputs === null) {
129
- await status.waiting();
130
- return;
131
- }
132
- try {
133
- await this.#update(load, status, inputs, trigger, signal);
134
- }
135
- catch (error) {
136
- // A stop is not the dataset's failure.
137
- if (signal?.aborted !== true)
138
- await status.failed(errorMessage(error));
139
- throw error;
140
- }
120
+ await refreshComputed({
121
+ rule: this.#rule,
122
+ inputs: await this.#inputs(load.sql, connections),
123
+ prepare: (keys) => this.#prepare(load, status, keys, signal),
124
+ status,
125
+ trigger,
126
+ signal,
127
+ });
141
128
  }
142
- // Brings the output rows in step with inputs, and tells status how many
143
- // are due and how many settled.
144
- async #update(load, status, inputs, trigger, signal) {
129
+ // Makes the schema, brings the table to its declaration, removes the rows
130
+ // whose key has no input, and gives the output table to write. Each
131
+ // transaction takes the number of its commit at its first change, so one
132
+ // that changes no row takes none. The load has one session, so the commits
133
+ // of the calls take turns.
134
+ async #prepare(load, status, keys, signal) {
145
135
  const key = quote(this.#rule.key.name);
146
136
  const table = this.#qualified(this.#table);
147
- // The number of a transaction's commit, taken at its first change, so a
148
- // transaction that changes no row takes none.
149
137
  const commits = (sql) => {
150
138
  let number;
151
139
  return () => (number ??= load.number(sql));
@@ -157,7 +145,7 @@ export class ComputedDataset extends DerivedDataset {
157
145
  const rows = new Map();
158
146
  const gone = [];
159
147
  for (const row of await sql.unsafe(`SELECT ${key} AS "key", "input_hash", "rule", "status" FROM ${table}`))
160
- if (inputs.has(row.key))
148
+ if (keys.has(row.key))
161
149
  rows.set(row.key, {
162
150
  inputHash: row.input_hash,
163
151
  rule: row.rule,
@@ -175,120 +163,39 @@ export class ComputedDataset extends DerivedDataset {
175
163
  // Writes a row in the commit that commit numbers, unless the stored row
176
164
  // has the same values: then it keeps its commit. The history keeps the
177
165
  // row it replaces.
178
- const store = async (sql, commit, id, inputHash, rule, status, error, values) => {
166
+ const store = async (sql, commit, output) => {
179
167
  const row = [
180
- id,
181
- inputHash,
182
- rule,
183
- status,
184
- error,
185
- ...Object.keys(this.#rule.output).map((column) => values[column] ?? null),
168
+ output.key,
169
+ output.inputHash,
170
+ output.rule,
171
+ output.status,
172
+ output.error,
173
+ ...Object.keys(this.#rule.output).map((column) => output.values[column] ?? null),
186
174
  ];
187
175
  const version = rowHash(hashed, row);
188
- const [before] = await sql.unsafe(`SELECT "_elt_row_hash" AS "hash" FROM ${table} WHERE ${key} = $1`, [id]);
176
+ const [before] = await sql.unsafe(`SELECT "_elt_row_hash" AS "hash" FROM ${table} WHERE ${key} = $1`, [output.key]);
189
177
  if (before?.hash === version)
190
178
  return;
191
179
  const number = await commit();
192
180
  if (before !== undefined)
193
- await this.#history.keep(sql, number, table, `FROM ${table} AS "_elt_old" WHERE "_elt_old".${key} = $1`, [id]);
181
+ await this.#history.keep(sql, number, table, `FROM ${table} AS "_elt_old" WHERE "_elt_old".${key} = $1`, [output.key]);
194
182
  await sql.unsafe(`INSERT INTO ${table} (${written.join(', ')}) VALUES (${written.map((_, index) => `$${index + 1}`).join(', ')}) ON CONFLICT (${key}) DO UPDATE SET ${written
195
183
  .slice(1)
196
184
  .map((column) => `${column} = excluded.${column}`)
197
185
  .join(', ')}`, [...row.map(parameter), number, version]);
198
186
  };
199
- // The load has one session, so the commits of the calls take turns.
200
187
  const session = new Lock();
201
188
  const settle = (work) => session.run(() => load.transaction(work), { signal });
202
- const due = [];
203
- for (const [id, input] of inputs) {
204
- if (signal?.aborted)
205
- return;
206
- const inputHash = hash('sha256', JSON.stringify(input));
207
- const rule = await this.#rule.identity(input);
208
- const before = stored.get(id);
209
- // A rule that does not handle the input keeps what it made before.
210
- if (rule === null) {
211
- if (before?.status === 'computed')
212
- continue;
213
- if (before?.rule === 'unsupported' && before.inputHash === inputHash)
214
- continue;
215
- await settle((sql) => store(sql, commits(sql), id, inputHash, 'unsupported', 'empty', null, {}));
216
- continue;
217
- }
218
- if (before !== undefined &&
219
- before.inputHash === inputHash &&
220
- before.rule === rule) {
221
- // A failed input waits for the refresh after the passes.
222
- if (before.status !== 'failed' || trigger.kind === 'commit')
223
- continue;
224
- }
225
- // The passes try each input, as it is now, at most once: the refresh
226
- // after them leaves what a commit refresh of the same passes tried.
227
- const attempt = `${id}\0${inputHash}\0${rule}`;
228
- if (trigger.tried.has(attempt))
229
- continue;
230
- if (trigger.kind === 'commit')
231
- trigger.tried.add(attempt);
232
- due.push({ id, input, inputHash, rule });
233
- }
234
- await status.refreshing({
235
- ...(await this.#counts(load.sql, inputs)),
236
- pending: due.length,
237
- });
238
- // A pool of calls, each taking the next batch once its own committed, so
239
- // a slow call holds only its place, as a worker pool keeps its other
240
- // workers busy. A call that fails stops the pool taking more.
241
- const failures = [];
242
- let next = 0;
243
- await Promise.all(Array.from({ length: this.#rule.concurrency }, async () => {
244
- while (failures.length === 0 && signal?.aborted !== true) {
245
- const batch = due.slice(next, next + this.#rule.batch);
246
- next += this.#rule.batch;
247
- if (batch.length === 0)
248
- return;
249
- let results;
250
- try {
251
- results = await this.#rule.compute(batch.map(({ input }) => input), signal);
252
- }
253
- catch (error) {
254
- failures.push(error);
255
- return;
256
- }
257
- // Each call commits on its own, so readers see rows as they land.
258
- await settle(async (sql) => {
259
- const commit = commits(sql);
260
- for (const [index, { id, inputHash, rule }] of batch.entries()) {
261
- const result = results[index];
262
- if (result === undefined || 'gone' in result)
263
- continue;
264
- if ('value' in result)
265
- await store(sql, commit, id, inputHash, rule, 'computed', null, result.value);
266
- else if ('error' in result)
267
- await store(sql, commit, id, inputHash, rule, 'failed', result.error, {});
268
- else
269
- await store(sql, commit, id, inputHash, rule, 'empty', null, {});
270
- }
271
- await status.settled(sql, batch.length);
272
- });
273
- }
274
- }));
275
- if (failures.length > 0)
276
- throw failures[0];
277
- if (signal?.aborted === true)
278
- return;
279
- await status.current({
280
- ...(await this.#counts(load.sql, inputs)),
281
- pending: 0,
282
- });
283
- }
284
- // How many inputs there are, and how many output rows have each status.
285
- async #counts(sql, inputs) {
286
- const counted = new Map((await sql.unsafe(`SELECT "status", count(*)::int AS "n" FROM ${this.#qualified(this.#table)} GROUP BY "status"`)).map(({ status, n }) => [status, n]));
287
189
  return {
288
- inputs: inputs.size,
289
- computed: counted.get('computed') ?? 0,
290
- empty: counted.get('empty') ?? 0,
291
- failed: counted.get('failed') ?? 0,
190
+ stored,
191
+ store: (output) => settle((sql) => store(sql, commits(sql), output)),
192
+ settle: (outputs, settled) => settle(async (sql) => {
193
+ const commit = commits(sql);
194
+ for (const output of outputs)
195
+ await store(sql, commit, output);
196
+ await status.settled(sql, settled);
197
+ }),
198
+ counts: async () => new Map((await load.sql.unsafe(`SELECT "status", count(*)::int AS "n" FROM ${table} GROUP BY "status"`)).map(({ status, n }) => [status, n])),
292
199
  };
293
200
  }
294
201
  // Makes the table, its history and its view, or makes the table again when
package/dist/index.d.ts CHANGED
@@ -4,7 +4,8 @@ export { PostgresCheckpointStore } from './postgres-checkpoint-store.ts';
4
4
  export { PostgresColumn } from './postgres-column.ts';
5
5
  export { PostgresColumns } from './postgres-columns.ts';
6
6
  export { PostgresDestination } from './postgres-destination.ts';
7
- export { type PassState, type PassStatus, readPassStatus, type StreamPassStatus, } from './postgres-passes.ts';
7
+ export type { PassState, PassStatus, StreamPassStatus } from '@eltjs/elt';
8
+ export { readPassStatus } from './postgres-passes.ts';
8
9
  export { PostgresSyncHistory } from './postgres-sync-history.ts';
9
10
  export { PostgresTable } from './postgres-table.ts';
10
11
  export { View } from './view.ts';
package/dist/index.js CHANGED
@@ -3,7 +3,7 @@ export { PostgresCheckpointStore } from './postgres-checkpoint-store.js';
3
3
  export { PostgresColumn } from './postgres-column.js';
4
4
  export { PostgresColumns } from './postgres-columns.js';
5
5
  export { PostgresDestination } from './postgres-destination.js';
6
- export { readPassStatus, } from './postgres-passes.js';
6
+ export { readPassStatus } from './postgres-passes.js';
7
7
  export { PostgresSyncHistory } from './postgres-sync-history.js';
8
8
  export { PostgresTable } from './postgres-table.js';
9
9
  export { View } from './view.js';
@@ -1,4 +1,5 @@
1
- import { committedAt, expiry, numberingIn, parsePosition, settledMark, } from './postgres-numbering.js';
1
+ import { parsePosition, readingExpiry } from '@eltjs/elt';
2
+ import { committedAt, numberingIn, postgresDatabase, settledMark, } from './postgres-numbering.js';
2
3
  import { PostgresSession } from './postgres-session.js';
3
4
  // Reads the database as of the position, or as of its newest position. The
4
5
  // high-water mark is taken before the snapshot, so the snapshot sees each
@@ -8,7 +9,9 @@ import { PostgresSession } from './postgres-session.js';
8
9
  // snapshot sees, the transaction sets elt.position, which each reader view
9
10
  // reads; no read makes or changes an object.
10
11
  export async function readAsOf(url, position) {
11
- const asked = position === undefined ? undefined : parsePosition(position);
12
+ const asked = position === undefined
13
+ ? undefined
14
+ : parsePosition(position, postgresDatabase);
12
15
  await using stack = new AsyncDisposableStack();
13
16
  const { sql } = stack.use(new PostgresSession(url, 'elt-reading'));
14
17
  const settled = await settledMark(sql);
@@ -20,10 +23,9 @@ export async function readAsOf(url, position) {
20
23
  });
21
24
  const numbered = await numberingIn(sql, settled);
22
25
  if (asked !== undefined) {
23
- const expired = expiry(numbered, asked);
26
+ const expired = readingExpiry(numbered, asked, postgresDatabase, 'the database');
24
27
  if (expired !== undefined)
25
28
  return { expired };
26
- refuseInsidePass(numbered, asked);
27
29
  }
28
30
  const number = asked?.number ?? numbered?.ends.at(-1);
29
31
  const commit = number === undefined ? undefined : await committedAt(sql, number);
@@ -43,9 +45,3 @@ export async function readAsOf(url, position) {
43
45
  [Symbol.asyncDispose]: () => owned.disposeAsync(),
44
46
  };
45
47
  }
46
- // No later load can straddle a position, so one that a reader was given
47
- // stays one; a commit that is not a position stays inside its pass.
48
- function refuseInsidePass(numbered, { epoch, number }) {
49
- if (!numbered?.ends.includes(number))
50
- throw new TypeError(`${epoch}:${number} is not a position of the database: a reading of a commit inside a pass would show a part of that pass. Read a position that positions() lists.`);
51
- }
@@ -1,20 +1,10 @@
1
1
  import type postgres from 'postgres';
2
- import type { PipelineClock } from '@eltjs/elt';
2
+ import { DerivedStatus, type PipelineClock } from '@eltjs/elt';
3
3
  import { type Queries } from './postgres-session.ts';
4
4
  export declare const derivedStatusTable = "CREATE TABLE IF NOT EXISTS _warehouse.derived_status (\n dataset text PRIMARY KEY,\n kind text NOT NULL CHECK (kind IN ('view', 'computed')),\n state text NOT NULL CHECK (state IN ('waiting', 'refreshing', 'current', 'failed')),\n as_of timestamptz, refreshed_at timestamptz, refreshing_since timestamptz, error text,\n inputs bigint, computed bigint, empty bigint, failed bigint, pending bigint\n )";
5
- export type DerivedCounts = {
6
- readonly inputs: number;
7
- readonly computed: number;
8
- readonly empty: number;
9
- readonly failed: number;
10
- readonly pending: number;
11
- };
12
- export declare class PostgresDerivedStatus {
5
+ export declare class PostgresDerivedStatus extends DerivedStatus {
13
6
  #private;
14
7
  constructor(sql: postgres.Sql, dataset: string, kind: 'view' | 'computed', clock: PipelineClock);
15
- waiting(): Promise<void>;
16
- refreshing(counts: DerivedCounts): Promise<void>;
17
8
  settled(sql: Queries, count: number): Promise<void>;
18
- current(counts?: DerivedCounts): Promise<void>;
19
- failed(error: string): Promise<void>;
9
+ protected write(values: Readonly<Record<string, string | number | null>>): Promise<void>;
20
10
  }
@@ -1,3 +1,4 @@
1
+ import { DerivedStatus } from '@eltjs/elt';
1
2
  import { lockSchema } from './postgres-session.js';
2
3
  // One row for each derived dataset of the database: what its refreshes left,
3
4
  // which readers see through marts.derived_status. The sync history makes it
@@ -10,56 +11,26 @@ export const derivedStatusTable = `CREATE TABLE IF NOT EXISTS _warehouse.derived
10
11
  as_of timestamptz, refreshed_at timestamptz, refreshing_since timestamptz, error text,
11
12
  inputs bigint, computed bigint, empty bigint, failed bigint, pending bigint
12
13
  )`;
13
- // What one refresh of a derived dataset tells readers, as Snowflake keeps the
14
- // state and DATA_TIMESTAMP of each dynamic table, and pgai the pending items
15
- // of each vectorizer. A refresh that begins keeps as_of, the data of the
16
- // last refresh that ended, until it ends too. dataset is the name with its
14
+ // The derived_status of the database, where dataset is the name with its
17
15
  // schema.
18
- export class PostgresDerivedStatus {
16
+ export class PostgresDerivedStatus extends DerivedStatus {
19
17
  #sql;
20
18
  #dataset;
21
19
  #kind;
22
- #clock;
23
- // When this refresh began to read, which its end gives as as_of.
24
- #since;
25
20
  // sql is a session outside any transaction, between the dataset's own.
26
21
  constructor(sql, dataset, kind, clock) {
22
+ super(clock);
27
23
  this.#sql = sql;
28
24
  this.#dataset = dataset;
29
25
  this.#kind = kind;
30
- this.#clock = clock;
31
- this.#since = this.#now();
32
- }
33
- waiting() {
34
- return this.#write({ state: 'waiting', refreshing_since: null });
35
- }
36
- refreshing(counts) {
37
- return this.#write({
38
- state: 'refreshing',
39
- refreshing_since: this.#since,
40
- ...counts,
41
- });
42
26
  }
43
27
  // Inside the transaction of a commit that settled count inputs.
44
28
  async settled(sql, count) {
45
29
  await sql.unsafe('UPDATE _warehouse.derived_status SET pending = GREATEST(pending - $1, 0) WHERE dataset = $2', [count, this.#dataset]);
46
30
  }
47
- current(counts) {
48
- return this.#write({
49
- state: 'current',
50
- as_of: this.#since,
51
- refreshed_at: this.#now(),
52
- refreshing_since: null,
53
- error: null,
54
- ...counts,
55
- });
56
- }
57
- failed(error) {
58
- return this.#write({ state: 'failed', refreshing_since: null, error });
59
- }
60
31
  // In a transaction of its own, which takes no commit number: the status is
61
32
  // no data of a position.
62
- async #write(values) {
33
+ async write(values) {
63
34
  const columns = ['dataset', 'kind', ...Object.keys(values)];
64
35
  const placeholders = columns.map((_, index) => `$${index + 1}`);
65
36
  const assignments = columns
@@ -71,7 +42,4 @@ export class PostgresDerivedStatus {
71
42
  await sql.unsafe(`INSERT INTO _warehouse.derived_status (${columns.join(', ')}) VALUES (${placeholders.join(', ')}) ON CONFLICT (dataset) DO UPDATE SET ${assignments.join(', ')}`, [this.#dataset, this.#kind, ...Object.values(values)]);
72
43
  });
73
44
  }
74
- #now() {
75
- return new Date(this.#clock.Date.now()).toISOString();
76
- }
77
45
  }
@@ -1,11 +1,11 @@
1
- import { Destination, } from '@eltjs/elt';
1
+ import { Destination, parsePosition, positionsAfter, } from '@eltjs/elt';
2
2
  import { quote } from './identifier.js';
3
3
  import { PostgresAppendWriter } from './postgres-append-writer.js';
4
4
  import { readAsOf } from './postgres-as-of.js';
5
5
  import { PostgresColumns } from './postgres-columns.js';
6
6
  import { PostgresDeduplicatingWriter } from './postgres-deduplicating-writer.js';
7
7
  import { PostgresLoad } from './postgres-load.js';
8
- import { positionsAfter } from './postgres-numbering.js';
8
+ import { postgresDatabase, readNumbering } from './postgres-numbering.js';
9
9
  import { PostgresOverwriteWriter } from './postgres-overwrite-writer.js';
10
10
  import { PostgresSession, connectionUrl, schemaName, } from './postgres-session.js';
11
11
  import { PostgresTable } from './postgres-table.js';
@@ -55,8 +55,9 @@ export class PostgresDestination extends Destination {
55
55
  // schema of the database is half read. A position of another epoch, or past
56
56
  // the newest commit, is expired.
57
57
  async positions(after) {
58
+ const since = after === undefined ? undefined : parsePosition(after, postgresDatabase);
58
59
  await using session = new PostgresSession(this.#url, 'elt-positions');
59
- return await positionsAfter(session.sql, after);
60
+ return positionsAfter(await readNumbering(session.sql), since, postgresDatabase);
60
61
  }
61
62
  // The database as of a position, or as of its newest one, read through the
62
63
  // reader views: { expired } for a position of another epoch or past the
@@ -1,6 +1,8 @@
1
1
  import type postgres from 'postgres';
2
+ import { type NumberedStore, type Numbering } from '@eltjs/elt';
2
3
  type Queries = postgres.Sql | postgres.TransactionSql;
3
4
  export declare function installNumbering(sql: postgres.Sql): Promise<void>;
5
+ export declare const postgresDatabase: NumberedStore;
4
6
  export type LoadOwner = {
5
7
  readonly connector: string;
6
8
  } | {
@@ -17,26 +19,11 @@ export declare function minValid(sql: Pick<postgres.Sql, 'unsafe'>): Promise<num
17
19
  export declare function releaseGate(sql: postgres.Sql): Promise<void>;
18
20
  export declare function commitOnce(sql: Queries, connector: string, at: Date, retention: Temporal.Duration): Promise<number>;
19
21
  export declare function expireBefore(sql: Pick<postgres.Sql, 'unsafe'>, changedIn: number, by: 'type' | 'clear'): Promise<void>;
20
- export type Numbering = {
21
- readonly epoch: string;
22
- readonly minValid: number;
23
- readonly expiredBy: 'retention' | 'type' | 'clear' | null;
24
- readonly settled: number;
25
- readonly ends: readonly number[];
26
- };
27
22
  export declare function settledMark(sql: postgres.Sql): Promise<number | null>;
28
23
  export declare function numberingIn(sql: Queries, settled: number): Promise<Numbering | null>;
29
24
  export declare function committedAt(sql: Queries, number: number): Promise<{
30
25
  readonly at: Date;
31
26
  readonly newest: number;
32
27
  } | undefined>;
33
- export declare function positionsAfter(sql: postgres.Sql, after: string | undefined): Promise<readonly string[] | {
34
- readonly expired: string;
35
- }>;
36
- export type Position = {
37
- readonly epoch: string;
38
- readonly number: number;
39
- };
40
- export declare function expiry(numbered: Numbering | null, { epoch, number }: Position): string | undefined;
41
- export declare function parsePosition(position: string): Position;
28
+ export declare function readNumbering(sql: postgres.Sql): Promise<Numbering | null>;
42
29
  export {};
@@ -1,3 +1,4 @@
1
+ import { numbering } from '@eltjs/elt';
1
2
  import { quote } from './identifier.js';
2
3
  import { lockSchema } from './postgres-session.js';
3
4
  // The schema that every destination of the database shares, beside the sync
@@ -53,6 +54,11 @@ export async function installNumbering(sql) {
53
54
  await transaction.unsafe(statement);
54
55
  });
55
56
  }
57
+ // How a Postgres database names itself in the reasons of its positions.
58
+ export const postgresDatabase = {
59
+ engine: 'Postgres',
60
+ noun: 'database',
61
+ };
56
62
  // The column of _elt_loads that names owner, and its name there.
57
63
  function owned(owner) {
58
64
  return 'connector' in owner
@@ -213,27 +219,25 @@ export async function settledMark(sql) {
213
219
  }
214
220
  }
215
221
  // The numbering as the snapshot of the transaction sql runs in sees it, with
216
- // settled taken before the snapshot. A position is a commit that ends a
217
- // closed load where no load straddles it, so a reader there sees no part of
218
- // a pass. Commits past the mark that the snapshot sees still reach across
219
- // the ends before them.
222
+ // settled taken before the snapshot. Commits past the mark that the snapshot
223
+ // sees still reach across the ends before them.
220
224
  export async function numberingIn(sql, settled) {
221
225
  const [row] = await sql `SELECT "epoch", "min_valid", "expired_by" FROM ${sql.unsafe(epochs)}`;
222
226
  if (row === undefined)
223
227
  return null;
224
- const spans = (await sql `
225
- SELECT min(c."number") AS "first",
226
- CASE WHEN l."closed_at" IS NULL THEN NULL ELSE max(c."number") END AS "last"
227
- FROM ${sql.unsafe(commits)} AS c
228
- JOIN ${sql.unsafe(loads)} AS l ON l."id" = c."load"
229
- GROUP BY l."id" ORDER BY "first"`).map(({ first, last }) => ({ first, last: last ?? Infinity }));
230
- return {
228
+ const spans = await sql `
229
+ SELECT min(c."number") AS "first",
230
+ CASE WHEN l."closed_at" IS NULL THEN NULL ELSE max(c."number") END AS "last"
231
+ FROM ${sql.unsafe(commits)} AS c
232
+ JOIN ${sql.unsafe(loads)} AS l ON l."id" = c."load"
233
+ GROUP BY l."id"`;
234
+ return numbering({
231
235
  epoch: row.epoch,
232
236
  minValid: row.min_valid,
233
237
  expiredBy: row.expired_by,
234
238
  settled,
235
- ends: ends(spans, settled).filter((end) => end >= row.min_valid),
236
- };
239
+ spans,
240
+ });
237
241
  }
238
242
  // When the commit numbered number was made, and the newest number that the
239
243
  // snapshot of the transaction sql runs in sees.
@@ -243,69 +247,11 @@ export async function committedAt(sql, number) {
243
247
  FROM ${sql.unsafe(commits)} WHERE "number" = ${number}`;
244
248
  return row;
245
249
  }
246
- async function readNumbering(sql) {
250
+ // The numbering in a snapshot of its own, with the high-water mark taken
251
+ // before it, or null when the database has none.
252
+ export async function readNumbering(sql) {
247
253
  const settled = await settledMark(sql);
248
254
  if (settled === null)
249
255
  return null;
250
256
  return sql.begin('ISOLATION LEVEL REPEATABLE READ READ ONLY', (transaction) => numberingIn(transaction, settled));
251
257
  }
252
- // A sweep line over the loads in order of their first commits: an end is a
253
- // position when no load that began at or before it ends after it. An open
254
- // load has no end yet, so it reaches past every later commit.
255
- function ends(spans, settled) {
256
- const found = [];
257
- const pending = spans.values();
258
- let span = pending.next();
259
- let reach = 0;
260
- for (const end of spans
261
- .map(({ last }) => last)
262
- .filter((last) => last <= settled)
263
- .sort((left, right) => left - right)) {
264
- for (; !span.done && span.value.first <= end; span = pending.next())
265
- reach = Math.max(reach, span.value.last);
266
- if (reach === end)
267
- found.push(end);
268
- }
269
- return found;
270
- }
271
- // The positions of the database after `after`, ascending, or all of them. A
272
- // position of another epoch, or past the newest settled commit, is expired.
273
- export async function positionsAfter(sql, after) {
274
- const since = after === undefined ? undefined : parsePosition(after);
275
- const numbered = await readNumbering(sql);
276
- if (since === undefined)
277
- return numbered === null
278
- ? []
279
- : numbered.ends.map((end) => `${numbered.epoch}:${end}`);
280
- const expired = expiry(numbered, since);
281
- if (expired !== undefined)
282
- return { expired };
283
- return (numbered?.ends ?? [])
284
- .filter((end) => end > since.number)
285
- .map((end) => `${since.epoch}:${end}`);
286
- }
287
- export function expiry(numbered, { epoch, number }) {
288
- const position = `${epoch}:${number}`;
289
- if (numbered === null)
290
- return `Position ${position} is of commits this database does not have`;
291
- if (epoch !== numbered.epoch)
292
- return `Position ${position} is of an earlier numbering: the commits of this database were numbered again since`;
293
- if (number > numbered.settled)
294
- return `Position ${position} is past the newest commit of this database, ${numbered.settled}`;
295
- if (number < numbered.minValid)
296
- switch (numbered.expiredBy) {
297
- case 'type':
298
- return `Position ${position} is before commit ${numbered.minValid}, which changed the type of a column to one that does not hold each earlier value: the history keeps no row that only a reading before that commit reads`;
299
- case 'clear':
300
- return `Position ${position} is before commit ${numbered.minValid}, a clear that forgot the rows of its table: the history keeps no row that only a reading before it reads`;
301
- default:
302
- return `Position ${position} is before commit ${numbered.minValid}, the oldest position that the retention of the database keeps readable: the history keeps no row that only a reading before it reads`;
303
- }
304
- return undefined;
305
- }
306
- export function parsePosition(position) {
307
- const groups = /^(?<epoch>[^:]+):(?<number>0|[1-9]\d*)$/.exec(position)?.groups;
308
- if (groups?.epoch === undefined || groups.number === undefined)
309
- throw new TypeError(`${position} is not a position of a Postgres database`);
310
- return { epoch: groups.epoch, number: Number(groups.number) };
311
- }
@@ -1,44 +1,5 @@
1
- import type { FailureType, SyncStatus } from '@eltjs/elt';
2
- export type PassState = SyncStatus | 'running';
3
- type Started = {
4
- readonly startedAt: string;
5
- readonly lastSucceededAt: string | null;
6
- readonly currentAsOf: string | null;
7
- };
8
- export type PassStatus = (Started & {
9
- readonly state: 'running';
10
- readonly completedAt: null;
11
- readonly error: null;
12
- readonly failureType: null;
13
- }) | (Started & {
14
- readonly state: 'succeeded';
15
- readonly completedAt: string;
16
- readonly error: null;
17
- readonly failureType: null;
18
- }) | (Started & {
19
- readonly state: 'cancelled';
20
- readonly completedAt: string;
21
- readonly error: string;
22
- readonly failureType: null;
23
- }) | (Started & {
24
- readonly state: 'partial' | 'failed';
25
- readonly completedAt: string;
26
- readonly error: string;
27
- readonly failureType: FailureType;
28
- });
29
- export type StreamPassStatus = {
30
- readonly stream: string;
31
- readonly lastSucceededAt: string | null;
32
- readonly currentAsOf: string | null;
33
- } & ({
34
- readonly state: 'running' | 'succeeded' | 'cancelled';
35
- readonly error: null;
36
- } | {
37
- readonly state: 'partial' | 'failed';
38
- readonly error: string;
39
- });
1
+ import { type PassStatus, type StreamPassStatus } from '@eltjs/elt';
40
2
  export declare function readPassStatus(url: string): Promise<{
41
3
  readonly pass: PassStatus | null;
42
4
  readonly streams: readonly StreamPassStatus[];
43
5
  }>;
44
- export {};
@@ -1,3 +1,4 @@
1
+ import { combinedPass, } from '@eltjs/elt';
1
2
  import { PostgresSession } from './postgres-session.js';
2
3
  // The latest pass of the connections that the sync history of a database
3
4
  // records, and each stream's latest outcome, with the errors of its failures
@@ -15,7 +16,7 @@ export async function readPassStatus(url) {
15
16
  FROM marts.sync_status`).map((row) => recorded(row, passState(row.status)));
16
17
  const streams = await streamOutcomes(sql);
17
18
  return {
18
- pass: first === undefined ? null : combined([first, ...others]),
19
+ pass: first === undefined ? null : combinedPass([first, ...others]),
19
20
  streams,
20
21
  };
21
22
  });
@@ -95,70 +96,6 @@ function recorded(row, state) {
95
96
  };
96
97
  }
97
98
  }
98
- // The latest passes of the connections of one database as one pass, which
99
- // is only as good as its worst connection: it runs while any of them runs,
100
- // began when the first began, ended when the last ended, and is current only
101
- // as of the least current. A failure of one connection makes the pass
102
- // partial, not succeeded, and its error keeps the error of each connection
103
- // that did not succeed, as Go's errors.Join does, one on each line.
104
- function combined(passes) {
105
- const started = {
106
- startedAt: earliest(passes.map(({ startedAt }) => startedAt)),
107
- lastSucceededAt: earliestOfAll(passes.map(({ lastSucceededAt }) => lastSucceededAt)),
108
- currentAsOf: earliestOfAll(passes.map(({ currentAsOf }) => currentAsOf)),
109
- };
110
- const ended = passes.filter((pass) => pass.state !== 'running');
111
- if (ended.length < passes.length)
112
- return {
113
- ...started,
114
- state: 'running',
115
- completedAt: null,
116
- error: null,
117
- failureType: null,
118
- };
119
- const completedAt = latest(ended.map(({ completedAt }) => completedAt));
120
- const error = ended
121
- .flatMap((pass) => (pass.error === null ? [] : [pass.error]))
122
- .join('\n');
123
- if (ended.some(({ state }) => state === 'cancelled'))
124
- return {
125
- ...started,
126
- state: 'cancelled',
127
- completedAt,
128
- error,
129
- failureType: null,
130
- };
131
- if (ended.every(({ state }) => state === 'succeeded'))
132
- return {
133
- ...started,
134
- state: 'succeeded',
135
- completedAt,
136
- error: null,
137
- failureType: null,
138
- };
139
- return {
140
- ...started,
141
- state: ended.every(({ state }) => state === 'failed')
142
- ? 'failed'
143
- : 'partial',
144
- completedAt,
145
- error,
146
- failureType: ended.some(({ failureType }) => failureType === 'config')
147
- ? 'config'
148
- : 'system',
149
- };
150
- }
151
- function earliest(instants) {
152
- return instants.reduce((first, next) => Date.parse(next) < Date.parse(first) ? next : first);
153
- }
154
- function latest(instants) {
155
- return instants.reduce((last, next) => Date.parse(next) > Date.parse(last) ? next : last);
156
- }
157
- // The earliest instant, or null when any of them is null.
158
- function earliestOfAll(instants) {
159
- const known = instants.filter((instant) => instant !== null);
160
- return known.length < instants.length ? null : earliest(known);
161
- }
162
99
  // A timestamptz as the ISO text that elt-sqlite stores and gives.
163
100
  function instant(value) {
164
101
  if (value instanceof Date)
package/dist/view.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import postgres from 'postgres';
2
- import { DerivedDataset, errorMessage, } from '@eltjs/elt';
2
+ import { DerivedDataset, refreshView, } from '@eltjs/elt';
3
3
  import { PostgresDerivedStatus } from './postgres-derived-status.js';
4
4
  import { PostgresDestination } from './postgres-destination.js';
5
5
  import { PostgresSession } from './postgres-session.js';
@@ -8,11 +8,7 @@ import { checkView, publishPostgresViews, } from './postgres-views.js';
8
8
  // data that Postgres computes on each read, as a dbt model materialized as a
9
9
  // view is. It lives in the schema of into, where its SQL's bare names resolve
10
10
  // too. Its inputs are the relations its SQL reads, so it names none of them.
11
- // It appears once its query compiles, so a reader never finds a view that
12
- // fails; until a copy of the pipeline has made each relation it reads, and
13
- // each derived dataset before it that it reads has appeared, it waits, as dbt
14
- // skips a model whose upstream model was skipped. Any other error of its
15
- // query is the view's own failure.
11
+ // It appears, or waits, as refreshView of elt says.
16
12
  export class View extends DerivedDataset {
17
13
  name;
18
14
  // The database and schema the view lives in, where a computed dataset
@@ -36,25 +32,20 @@ export class View extends DerivedDataset {
36
32
  }
37
33
  async refresh({ connections, earlier, clock, }) {
38
34
  await using session = new PostgresSession(this.into.url, 'elt-derived');
39
- const status = new PostgresDerivedStatus(session.sql, this.qualifiedName, 'view', clock);
40
- try {
41
- await session.sql.begin((transaction) => publishPostgresViews(transaction, {
42
- schema: this.into.schema,
43
- views: [this.#view],
44
- }));
45
- }
46
- catch (error) {
47
- const missing = missingRelation(error, this.into.schema);
48
- if (missing !== undefined &&
49
- (made(connections, this.into).has(missing) ||
50
- earlier.some((dataset) => derivedRelation(dataset) === missing))) {
51
- await status.waiting();
52
- return;
53
- }
54
- await status.failed(errorMessage(error));
55
- throw error;
56
- }
57
- await status.current();
35
+ await refreshView({
36
+ publish: async () => {
37
+ await session.sql.begin((transaction) => publishPostgresViews(transaction, {
38
+ schema: this.into.schema,
39
+ views: [this.#view],
40
+ }));
41
+ },
42
+ missing: (error) => missingRelation(error, this.into.schema),
43
+ awaited: new Set([
44
+ ...made(connections, this.into),
45
+ ...earlier.flatMap((dataset) => derivedRelation(dataset) ?? []),
46
+ ]),
47
+ status: new PostgresDerivedStatus(session.sql, this.qualifiedName, 'view', clock),
48
+ });
58
49
  }
59
50
  }
60
51
  // The schema-qualified relation that Postgres found missing, with a bare name
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@eltjs/elt-postgresql",
3
- "version": "0.4.1",
3
+ "version": "0.4.2",
4
4
  "description": "The Postgres destination of elt: each stream is a typed table in one schema, with a reader view in another, a history of each row for readings as of a position, and the views and computed datasets that a pipeline derives.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -37,7 +37,7 @@
37
37
  "access": "public"
38
38
  },
39
39
  "dependencies": {
40
- "@eltjs/elt": "^0.4.1",
40
+ "@eltjs/elt": "^0.4.2",
41
41
  "@zukhruf/async": "^0.8.31",
42
42
  "postgres": "^3.4.9"
43
43
  },