@hazeljs/data 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connectors/__tests__/jsonl.connector.test.d.ts +2 -0
- package/dist/connectors/__tests__/jsonl.connector.test.d.ts.map +1 -0
- package/dist/connectors/__tests__/jsonl.connector.test.js +261 -0
- package/dist/connectors/jsonl.connector.d.ts +51 -0
- package/dist/connectors/jsonl.connector.d.ts.map +1 -0
- package/dist/connectors/jsonl.connector.js +100 -0
- package/dist/connectors/postgres.connector.d.ts +78 -0
- package/dist/connectors/postgres.connector.d.ts.map +1 -0
- package/dist/connectors/postgres.connector.js +224 -0
- package/dist/contracts/__tests__/contract-registry.test.d.ts +2 -0
- package/dist/contracts/__tests__/contract-registry.test.d.ts.map +1 -0
- package/dist/contracts/__tests__/contract-registry.test.js +770 -0
- package/dist/contracts/__tests__/contract.decorator.test.d.ts +2 -0
- package/dist/contracts/__tests__/contract.decorator.test.d.ts.map +1 -0
- package/dist/contracts/__tests__/contract.decorator.test.js +177 -0
- package/dist/contracts/contract-registry.d.ts +57 -0
- package/dist/contracts/contract-registry.d.ts.map +1 -0
- package/dist/contracts/contract-registry.js +285 -0
- package/dist/contracts/contract.decorator.d.ts +70 -0
- package/dist/contracts/contract.decorator.d.ts.map +1 -0
- package/dist/contracts/contract.decorator.js +55 -0
- package/dist/contracts/contract.types.d.ts +65 -0
- package/dist/contracts/contract.types.d.ts.map +1 -0
- package/dist/contracts/contract.types.js +5 -0
- package/dist/contracts/index.d.ts +9 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +16 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -2
- package/dist/testing/schema-faker.test.js +33 -0
- package/dist/transformers/transformer.service.test.js +40 -0
- package/package.json +2 -2
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PostgresSink = exports.PostgresSource = void 0;
|
|
4
|
+
class PostgresSource {
|
|
5
|
+
constructor(options) {
|
|
6
|
+
this.pool = null;
|
|
7
|
+
this.name = options.name ?? `postgres:${options.database}.${options.table}`;
|
|
8
|
+
this.options = { batchSize: 1000, ...options };
|
|
9
|
+
}
|
|
10
|
+
async ensurePool() {
|
|
11
|
+
if (this.pool)
|
|
12
|
+
return;
|
|
13
|
+
try {
|
|
14
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
15
|
+
const { Pool } = require('pg');
|
|
16
|
+
this.pool = new Pool({
|
|
17
|
+
host: this.options.host,
|
|
18
|
+
port: this.options.port,
|
|
19
|
+
database: this.options.database,
|
|
20
|
+
user: this.options.user,
|
|
21
|
+
password: this.options.password,
|
|
22
|
+
max: 10,
|
|
23
|
+
idleTimeoutMillis: 30000,
|
|
24
|
+
connectionTimeoutMillis: 5000,
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
throw new Error(`PostgresSource requires 'pg' package. Install with: npm install pg\nError: ${error instanceof Error ? error.message : String(error)}`);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async open() {
|
|
32
|
+
await this.ensurePool();
|
|
33
|
+
if (!this.pool)
|
|
34
|
+
throw new Error('Failed to initialize pool');
|
|
35
|
+
// Test connection
|
|
36
|
+
const client = await this.pool.connect();
|
|
37
|
+
try {
|
|
38
|
+
await client.query('SELECT 1');
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
client.release();
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
async close() {
|
|
45
|
+
if (this.pool) {
|
|
46
|
+
await this.pool.end();
|
|
47
|
+
this.pool = null;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async readAll() {
|
|
51
|
+
const records = [];
|
|
52
|
+
for await (const record of this.read()) {
|
|
53
|
+
records.push(record);
|
|
54
|
+
}
|
|
55
|
+
return records;
|
|
56
|
+
}
|
|
57
|
+
async *read() {
|
|
58
|
+
await this.ensurePool();
|
|
59
|
+
if (!this.pool)
|
|
60
|
+
throw new Error('Failed to initialize pool');
|
|
61
|
+
const columns = this.options.columns ? this.options.columns.join(', ') : '*';
|
|
62
|
+
let query = `SELECT ${columns} FROM ${this.options.table}`;
|
|
63
|
+
if (this.options.where) {
|
|
64
|
+
query += ` WHERE ${this.options.where}`;
|
|
65
|
+
}
|
|
66
|
+
if (this.options.orderBy) {
|
|
67
|
+
query += ` ORDER BY ${this.options.orderBy}`;
|
|
68
|
+
}
|
|
69
|
+
const client = await this.pool.connect();
|
|
70
|
+
try {
|
|
71
|
+
const result = await client.query(query);
|
|
72
|
+
for (const row of result.rows) {
|
|
73
|
+
yield row;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
client.release();
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Execute a custom SQL query
|
|
82
|
+
*/
|
|
83
|
+
async query(sql, params) {
|
|
84
|
+
await this.ensurePool();
|
|
85
|
+
if (!this.pool)
|
|
86
|
+
throw new Error('Failed to initialize pool');
|
|
87
|
+
const client = await this.pool.connect();
|
|
88
|
+
try {
|
|
89
|
+
const result = await client.query(sql, params);
|
|
90
|
+
return result.rows;
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
client.release();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
exports.PostgresSource = PostgresSource;
|
|
98
|
+
/**
|
|
99
|
+
* PostgreSQL data sink — writes records to a Postgres table.
|
|
100
|
+
* Supports batch inserts and upserts.
|
|
101
|
+
*
|
|
102
|
+
* @example
|
|
103
|
+
* const sink = new PostgresSink({
|
|
104
|
+
* host: 'localhost',
|
|
105
|
+
* port: 5432,
|
|
106
|
+
* database: 'mydb',
|
|
107
|
+
* user: 'user',
|
|
108
|
+
* password: 'pass',
|
|
109
|
+
* table: 'users',
|
|
110
|
+
* columns: ['id', 'name', 'email'],
|
|
111
|
+
* conflictColumn: 'id', // for upserts
|
|
112
|
+
* });
|
|
113
|
+
* await sink.open();
|
|
114
|
+
* await sink.writeBatch(records);
|
|
115
|
+
* await sink.close();
|
|
116
|
+
*/
|
|
117
|
+
class PostgresSink {
|
|
118
|
+
constructor(options) {
|
|
119
|
+
this.pool = null;
|
|
120
|
+
this.buffer = [];
|
|
121
|
+
this.name = options.name ?? `postgres:${options.database}.${options.table}`;
|
|
122
|
+
this.options = { batchSize: 100, ...options };
|
|
123
|
+
}
|
|
124
|
+
async ensurePool() {
|
|
125
|
+
if (this.pool)
|
|
126
|
+
return;
|
|
127
|
+
try {
|
|
128
|
+
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
129
|
+
const { Pool } = require('pg');
|
|
130
|
+
this.pool = new Pool({
|
|
131
|
+
host: this.options.host,
|
|
132
|
+
port: this.options.port,
|
|
133
|
+
database: this.options.database,
|
|
134
|
+
user: this.options.user,
|
|
135
|
+
password: this.options.password,
|
|
136
|
+
max: 10,
|
|
137
|
+
idleTimeoutMillis: 30000,
|
|
138
|
+
connectionTimeoutMillis: 5000,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
throw new Error(`PostgresSink requires 'pg' package. Install with: npm install pg\nError: ${error instanceof Error ? error.message : String(error)}`);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async open() {
|
|
146
|
+
await this.ensurePool();
|
|
147
|
+
}
|
|
148
|
+
async close() {
|
|
149
|
+
if (this.buffer.length > 0) {
|
|
150
|
+
await this.flush();
|
|
151
|
+
}
|
|
152
|
+
if (this.pool) {
|
|
153
|
+
await this.pool.end();
|
|
154
|
+
this.pool = null;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
async write(record) {
|
|
158
|
+
this.buffer.push(record);
|
|
159
|
+
if (this.buffer.length >= (this.options.batchSize ?? 100)) {
|
|
160
|
+
await this.flush();
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
async writeBatch(records) {
|
|
164
|
+
this.buffer.push(...records);
|
|
165
|
+
await this.flush();
|
|
166
|
+
}
|
|
167
|
+
async flush() {
|
|
168
|
+
if (this.buffer.length === 0)
|
|
169
|
+
return;
|
|
170
|
+
if (!this.pool)
|
|
171
|
+
throw new Error('Pool not initialized');
|
|
172
|
+
const client = await this.pool.connect();
|
|
173
|
+
try {
|
|
174
|
+
const columns = this.options.columns;
|
|
175
|
+
const columnList = columns.join(', ');
|
|
176
|
+
const placeholders = columns.map((_, i) => `$${i + 1}`).join(', ');
|
|
177
|
+
if (this.options.conflictColumn) {
|
|
178
|
+
// Upsert (INSERT ... ON CONFLICT DO UPDATE)
|
|
179
|
+
const updates = columns
|
|
180
|
+
.filter((c) => c !== this.options.conflictColumn)
|
|
181
|
+
.map((c) => `${c} = EXCLUDED.${c}`)
|
|
182
|
+
.join(', ');
|
|
183
|
+
const query = `
|
|
184
|
+
INSERT INTO ${this.options.table} (${columnList})
|
|
185
|
+
VALUES (${placeholders})
|
|
186
|
+
ON CONFLICT (${this.options.conflictColumn})
|
|
187
|
+
DO UPDATE SET ${updates}
|
|
188
|
+
`;
|
|
189
|
+
for (const record of this.buffer) {
|
|
190
|
+
const values = columns.map((col) => record[col] ?? null);
|
|
191
|
+
await client.query(query, values);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
// Plain insert
|
|
196
|
+
const query = `INSERT INTO ${this.options.table} (${columnList}) VALUES (${placeholders})`;
|
|
197
|
+
for (const record of this.buffer) {
|
|
198
|
+
const values = columns.map((col) => record[col] ?? null);
|
|
199
|
+
await client.query(query, values);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
finally {
|
|
204
|
+
client.release();
|
|
205
|
+
}
|
|
206
|
+
this.buffer = [];
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Execute a custom SQL command
|
|
210
|
+
*/
|
|
211
|
+
async execute(sql, params) {
|
|
212
|
+
await this.ensurePool();
|
|
213
|
+
if (!this.pool)
|
|
214
|
+
throw new Error('Failed to initialize pool');
|
|
215
|
+
const client = await this.pool.connect();
|
|
216
|
+
try {
|
|
217
|
+
await client.query(sql, params);
|
|
218
|
+
}
|
|
219
|
+
finally {
|
|
220
|
+
client.release();
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
exports.PostgresSink = PostgresSink;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contract-registry.test.d.ts","sourceRoot":"","sources":["../../../src/contracts/__tests__/contract-registry.test.ts"],"names":[],"mappings":""}
|