@hemia/db-connector 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +387 -14
- package/dist/hemia-db-connector.esm.js +98 -1
- package/dist/hemia-db-connector.js +99 -0
- package/dist/types/Core/ClickHouseConnector.d.ts +12 -0
- package/dist/types/DBConnector.d.ts +5 -3
- package/dist/types/abstract/OLAPConnector.d.ts +14 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/types/managers/OLAPConnectionManager.d.ts +6 -0
- package/dist/types/types/OLAPTypes.d.ts +5 -0
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @hemia/db-connector
|
|
2
2
|
|
|
3
|
-
Conector unificado para bases de datos SQL
|
|
3
|
+
Conector unificado para bases de datos SQL, NoSQL y OLAP, diseñado para facilitar operaciones comunes como conexión, transacciones, queries y manipulación de datos. Ideal para aplicaciones Node.js que requieren una interfaz abstracta para distintos motores de base de datos.
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
@@ -12,10 +12,50 @@ npm install @hemia/db-connector
|
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
|
15
|
-
##
|
|
15
|
+
## Bases de Datos Soportadas
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
### 🗄️ SQL (OLTP - Online Transaction Processing)
|
|
18
|
+
Bases de datos relacionales optimizadas para transacciones y operaciones CRUD.
|
|
19
|
+
|
|
20
|
+
* ✅ **MySQL** - Base de datos relacional de código abierto
|
|
21
|
+
* ✅ **PostgreSQL** - Base de datos relacional avanzada con soporte JSON
|
|
22
|
+
* ✅ **Microsoft SQL Server (MSSQL)** - Base de datos empresarial de Microsoft
|
|
23
|
+
|
|
24
|
+
**Casos de uso:**
|
|
25
|
+
- Sistemas transaccionales (e-commerce, banking)
|
|
26
|
+
- Aplicaciones empresariales (ERP, CRM)
|
|
27
|
+
- Gestión de usuarios y autenticación
|
|
28
|
+
- Aplicaciones que requieren integridad referencial
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
### 📄 NoSQL (Document Store)
|
|
33
|
+
Bases de datos orientadas a documentos para datos no estructurados.
|
|
34
|
+
|
|
35
|
+
* ✅ **MongoDB** - Base de datos de documentos JSON flexible y escalable
|
|
36
|
+
|
|
37
|
+
**Casos de uso:**
|
|
38
|
+
- Aplicaciones con esquemas flexibles
|
|
39
|
+
- Gestión de contenido (CMS)
|
|
40
|
+
- Perfiles de usuario complejos
|
|
41
|
+
- Catálogos de productos
|
|
42
|
+
- Logs y eventos en formato JSON
|
|
43
|
+
- Aplicaciones en tiempo real
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
### 📊 OLAP (Online Analytical Processing)
|
|
48
|
+
Bases de datos columnares optimizadas para análisis y consultas agregadas.
|
|
49
|
+
|
|
50
|
+
* ✅ **ClickHouse** - Base de datos columnar de alta velocidad para análisis en tiempo real
|
|
51
|
+
|
|
52
|
+
**Casos de uso:**
|
|
53
|
+
- Análisis de logs y trazas (APM, Observability)
|
|
54
|
+
- Métricas y monitoreo en tiempo real
|
|
55
|
+
- Business Intelligence y Data Warehousing
|
|
56
|
+
- Análisis de eventos y comportamiento de usuarios
|
|
57
|
+
- Procesamiento de grandes volúmenes de datos
|
|
58
|
+
- Dashboards analíticos de alto rendimiento
|
|
19
59
|
|
|
20
60
|
---
|
|
21
61
|
|
|
@@ -37,12 +77,14 @@ await mongo.connect();
|
|
|
37
77
|
const docs = await mongo.find(MyModel, { status: 'active' });
|
|
38
78
|
```
|
|
39
79
|
|
|
40
|
-
### SQL (MySQL/MSSQL)
|
|
80
|
+
### SQL (MySQL/MSSQL/PostgreSQL)
|
|
41
81
|
|
|
42
82
|
```ts
|
|
43
|
-
import {
|
|
83
|
+
import { DBConnector } from '@hemia/db-connector';
|
|
84
|
+
import { DBSQLType } from '@hemia/db-connector/types/DBSQLTypes';
|
|
44
85
|
|
|
45
|
-
const
|
|
86
|
+
const dbConnector = DBConnector.getInstance();
|
|
87
|
+
const sql = dbConnector.getEngine(DBSQLType.MySQL, {
|
|
46
88
|
host: 'localhost',
|
|
47
89
|
user: 'root',
|
|
48
90
|
password: 'pass',
|
|
@@ -52,25 +94,182 @@ const sql = new SequelizeSqlConnector({
|
|
|
52
94
|
|
|
53
95
|
await sql.connect();
|
|
54
96
|
const rows = await sql.select('users', { where: { active: true } });
|
|
97
|
+
|
|
98
|
+
// Transacciones
|
|
99
|
+
await sql.withTransaction(async (transaction) => {
|
|
100
|
+
await sql.insert('orders', { userId: 1, total: 100 }, { transaction });
|
|
101
|
+
await sql.update('users', { id: 1 }, { balance: 500 }, { transaction });
|
|
102
|
+
});
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### ClickHouse (OLAP)
|
|
106
|
+
|
|
107
|
+
```ts
|
|
108
|
+
import { DBConnector } from '@hemia/db-connector';
|
|
109
|
+
import { DBOLAPType } from '@hemia/db-connector/types/OLAPTypes';
|
|
110
|
+
|
|
111
|
+
const dbConnector = DBConnector.getInstance();
|
|
112
|
+
const clickhouse = dbConnector.getEngine(DBOLAPType.ClickHouse, {
|
|
113
|
+
host: 'http://localhost:8123',
|
|
114
|
+
user: 'default',
|
|
115
|
+
password: '',
|
|
116
|
+
database: 'analytics',
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
await clickhouse.connect();
|
|
120
|
+
|
|
121
|
+
// Insertar datos (JSONEachRow por defecto)
|
|
122
|
+
await clickhouse.insert({
|
|
123
|
+
table: 'traces',
|
|
124
|
+
values: [
|
|
125
|
+
{ trace_id: '123', service_name: 'api', duration_ms: 45 },
|
|
126
|
+
{ trace_id: '124', service_name: 'db', duration_ms: 120 }
|
|
127
|
+
]
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
// Insertar con formato CSV
|
|
131
|
+
await clickhouse.insert({
|
|
132
|
+
table: 'traces',
|
|
133
|
+
values: '125,"cache",30\n126,"queue",85',
|
|
134
|
+
format: 'CSV'
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
// Buscar con filtros
|
|
138
|
+
const traces = await clickhouse.find('traces', {
|
|
139
|
+
service_name: 'api',
|
|
140
|
+
error: false
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// Query personalizada para análisis
|
|
144
|
+
const stats = await clickhouse.query(`
|
|
145
|
+
SELECT
|
|
146
|
+
service_name,
|
|
147
|
+
count() as total_requests,
|
|
148
|
+
avg(duration_ms) as avg_duration,
|
|
149
|
+
max(duration_ms) as max_duration
|
|
150
|
+
FROM traces
|
|
151
|
+
WHERE start_time >= now() - INTERVAL 1 HOUR
|
|
152
|
+
GROUP BY service_name
|
|
153
|
+
ORDER BY total_requests DESC
|
|
154
|
+
`);
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## Uso con DBConnector (Recomendado)
|
|
160
|
+
|
|
161
|
+
El patrón recomendado es usar `DBConnector` como punto de entrada único:
|
|
162
|
+
|
|
163
|
+
```ts
|
|
164
|
+
import { DBConnector } from '@hemia/db-connector';
|
|
165
|
+
import { DBSQLType, DBNoSQLType, DBOLAPType } from '@hemia/db-connector';
|
|
166
|
+
|
|
167
|
+
const dbConnector = DBConnector.getInstance();
|
|
168
|
+
|
|
169
|
+
// Obtener motor SQL
|
|
170
|
+
const mysql = dbConnector.getEngine(DBSQLType.MySQL, credentials);
|
|
171
|
+
|
|
172
|
+
// Obtener motor NoSQL
|
|
173
|
+
const mongo = dbConnector.getEngine(DBNoSQLType.MongoDB, credentials);
|
|
174
|
+
|
|
175
|
+
// Obtener motor OLAP
|
|
176
|
+
const clickhouse = dbConnector.getEngine(DBOLAPType.ClickHouse, credentials);
|
|
55
177
|
```
|
|
56
178
|
|
|
57
179
|
---
|
|
58
180
|
|
|
59
181
|
## Características destacadas
|
|
60
182
|
|
|
61
|
-
### MongoDBConnector
|
|
183
|
+
### MongoDBConnector (NoSQL)
|
|
62
184
|
|
|
63
185
|
* Conexión y desconexión automáticas
|
|
64
186
|
* Transacciones con rollback (`withTransaction`)
|
|
65
187
|
* CRUD genérico (`create`, `find`, `findOne`, `update`, `delete`, etc.)
|
|
66
|
-
* `aggregate`, `findAllFromMultipleModels`
|
|
188
|
+
* Operaciones avanzadas: `aggregate`, `findAllFromMultipleModels`
|
|
189
|
+
* Soporte para modelos Mongoose
|
|
190
|
+
* Manejo automático de sesiones
|
|
67
191
|
|
|
68
|
-
|
|
192
|
+
**Ejemplo de uso avanzado:**
|
|
193
|
+
```ts
|
|
194
|
+
// Aggregate pipeline
|
|
195
|
+
const results = await mongo.aggregate(UserModel, [
|
|
196
|
+
{ $match: { active: true } },
|
|
197
|
+
{ $group: { _id: '$country', total: { $sum: 1 } } }
|
|
198
|
+
]);
|
|
199
|
+
|
|
200
|
+
// Transacciones
|
|
201
|
+
await mongo.withTransaction(async (session) => {
|
|
202
|
+
await mongo.create(UserModel, { name: 'John' }, session);
|
|
203
|
+
await mongo.update(UserModel, { _id: userId }, { status: 'active' }, session);
|
|
204
|
+
});
|
|
205
|
+
```
|
|
69
206
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
207
|
+
---
|
|
208
|
+
|
|
209
|
+
### SequelizeSqlConnector (SQL)
|
|
210
|
+
|
|
211
|
+
* Operaciones CRUD: `select`, `insert`, `update`, `delete`
|
|
212
|
+
* Transacciones con soporte completo de Sequelize (`withTransaction`)
|
|
213
|
+
* Query SQL libre con `query()` para consultas complejas
|
|
73
214
|
* Manejo de errores y mapeo de códigos SQLSTATE (`createDatabaseError`)
|
|
215
|
+
* Soporte para stored procedures
|
|
216
|
+
* Conexión a MySQL, PostgreSQL, MSSQL
|
|
217
|
+
|
|
218
|
+
**Ejemplo de uso avanzado:**
|
|
219
|
+
```ts
|
|
220
|
+
// Query con stored procedure
|
|
221
|
+
const result = await sql.query('CALL get_user_stats(:userId)', {
|
|
222
|
+
replacements: { userId: 123 },
|
|
223
|
+
type: QueryTypes.SELECT
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
// Transacciones
|
|
227
|
+
await sql.withTransaction(async (transaction) => {
|
|
228
|
+
await sql.insert('orders', orderData, { transaction });
|
|
229
|
+
await sql.update('inventory', { productId: 1 }, { stock: 10 }, { transaction });
|
|
230
|
+
});
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
### ClickHouseConnector (OLAP)
|
|
236
|
+
|
|
237
|
+
* Conexión optimizada para análisis de alto rendimiento
|
|
238
|
+
* Múltiples formatos de inserción: `JSONEachRow`, `CSV`, `TabSeparated`, etc.
|
|
239
|
+
* Método `find()` con filtros para búsquedas rápidas
|
|
240
|
+
* Query SQL libre para análisis complejos
|
|
241
|
+
* Ideal para procesamiento de grandes volúmenes de datos
|
|
242
|
+
* Soporte para columnas calculadas (MATERIALIZED)
|
|
243
|
+
|
|
244
|
+
**Ejemplo de uso avanzado:**
|
|
245
|
+
```ts
|
|
246
|
+
// Insertar millones de registros eficientemente
|
|
247
|
+
await clickhouse.insert({
|
|
248
|
+
table: 'events',
|
|
249
|
+
values: largeDataArray, // Array con millones de registros
|
|
250
|
+
format: 'JSONEachRow'
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
// Análisis en tiempo real
|
|
254
|
+
const metrics = await clickhouse.query(`
|
|
255
|
+
SELECT
|
|
256
|
+
toStartOfMinute(timestamp) as minute,
|
|
257
|
+
countIf(status = 200) as success,
|
|
258
|
+
countIf(status >= 400) as errors,
|
|
259
|
+
quantile(0.95)(response_time) as p95_latency
|
|
260
|
+
FROM http_logs
|
|
261
|
+
WHERE timestamp >= now() - INTERVAL 1 HOUR
|
|
262
|
+
GROUP BY minute
|
|
263
|
+
ORDER BY minute DESC
|
|
264
|
+
`);
|
|
265
|
+
|
|
266
|
+
// Búsqueda con filtros múltiples
|
|
267
|
+
const errorTraces = await clickhouse.find('traces', {
|
|
268
|
+
service_name: 'payment-api',
|
|
269
|
+
error: true,
|
|
270
|
+
duration_ms: 1000 // Mayor a 1 segundo
|
|
271
|
+
});
|
|
272
|
+
```
|
|
74
273
|
|
|
75
274
|
---
|
|
76
275
|
|
|
@@ -83,13 +282,187 @@ interface CredentialsConnection {
|
|
|
83
282
|
user?: string;
|
|
84
283
|
password?: string;
|
|
85
284
|
database: string;
|
|
86
|
-
dialect?: 'mysql' | 'mssql';
|
|
285
|
+
dialect?: 'mysql' | 'mssql' | 'postgres';
|
|
87
286
|
logging?: boolean;
|
|
88
287
|
}
|
|
89
288
|
```
|
|
90
289
|
|
|
91
290
|
---
|
|
92
291
|
|
|
292
|
+
## Arquitectura y Patrones de Uso
|
|
293
|
+
|
|
294
|
+
### Lambda Architecture: Combinando SQL, NoSQL y OLAP
|
|
295
|
+
|
|
296
|
+
Una arquitectura común es usar diferentes bases de datos para diferentes propósitos:
|
|
297
|
+
|
|
298
|
+
```ts
|
|
299
|
+
// 1. OLTP (MySQL/PostgreSQL) - Transacciones en tiempo real
|
|
300
|
+
const mysql = dbConnector.getEngine(DBSQLType.MySQL, mysqlCredentials);
|
|
301
|
+
await mysql.insert('orders', { userId: 123, amount: 99.99 });
|
|
302
|
+
|
|
303
|
+
// 2. NoSQL (MongoDB) - Datos flexibles y documentos
|
|
304
|
+
const mongo = dbConnector.getEngine(DBNoSQLType.MongoDB, mongoCredentials);
|
|
305
|
+
await mongo.create(ProductModel, {
|
|
306
|
+
name: 'Laptop',
|
|
307
|
+
specs: { ram: '16GB', cpu: 'i7' } // Esquema flexible
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
// 3. OLAP (ClickHouse) - Análisis y reportes
|
|
311
|
+
const clickhouse = dbConnector.getEngine(DBOLAPType.ClickHouse, clickhouseCredentials);
|
|
312
|
+
await clickhouse.insert({
|
|
313
|
+
table: 'order_analytics',
|
|
314
|
+
values: [{
|
|
315
|
+
order_id: orderId,
|
|
316
|
+
user_id: 123,
|
|
317
|
+
amount: 99.99,
|
|
318
|
+
timestamp: new Date()
|
|
319
|
+
}]
|
|
320
|
+
});
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
### Casos de Uso por Tipo de Base de Datos
|
|
324
|
+
|
|
325
|
+
#### 🗄️ SQL (MySQL/PostgreSQL/MSSQL)
|
|
326
|
+
**Cuándo usar:**
|
|
327
|
+
- Necesitas transacciones ACID
|
|
328
|
+
- Relaciones complejas entre entidades
|
|
329
|
+
- Integridad referencial crítica
|
|
330
|
+
- Consultas con JOINs
|
|
331
|
+
|
|
332
|
+
**Ejemplos:**
|
|
333
|
+
- Sistema de facturación
|
|
334
|
+
- Gestión de inventario
|
|
335
|
+
- Sistema de usuarios y permisos
|
|
336
|
+
- Carrito de compras
|
|
337
|
+
|
|
338
|
+
#### 📄 NoSQL (MongoDB)
|
|
339
|
+
**Cuándo usar:**
|
|
340
|
+
- Esquemas variables o en evolución
|
|
341
|
+
- Necesitas escalar horizontalmente
|
|
342
|
+
- Documentos JSON complejos
|
|
343
|
+
- Alto volumen de escrituras
|
|
344
|
+
|
|
345
|
+
**Ejemplos:**
|
|
346
|
+
- Catálogo de productos con atributos variables
|
|
347
|
+
- Perfiles de usuario con datos personalizados
|
|
348
|
+
- Sistema de logs y auditoría
|
|
349
|
+
- Gestión de contenido (CMS)
|
|
350
|
+
|
|
351
|
+
#### 📊 OLAP (ClickHouse)
|
|
352
|
+
**Cuándo usar:**
|
|
353
|
+
- Análisis de grandes volúmenes de datos
|
|
354
|
+
- Consultas agregadas complejas
|
|
355
|
+
- Necesitas latencias de milisegundos en análisis
|
|
356
|
+
- Datos de series temporales
|
|
357
|
+
|
|
358
|
+
**Ejemplos:**
|
|
359
|
+
- Observabilidad y APM (traces, logs, métricas)
|
|
360
|
+
- Análisis de comportamiento de usuarios
|
|
361
|
+
- Dashboards de BI en tiempo real
|
|
362
|
+
- Análisis de eventos y clickstream
|
|
363
|
+
- Monitoreo de infraestructura
|
|
364
|
+
|
|
365
|
+
---
|
|
366
|
+
|
|
367
|
+
## Ejemplo Completo: Sistema de E-commerce con Observabilidad
|
|
368
|
+
|
|
369
|
+
```ts
|
|
370
|
+
import { DBConnector, DBSQLType, DBNoSQLType, DBOLAPType } from '@hemia/db-connector';
|
|
371
|
+
|
|
372
|
+
const connector = DBConnector.getInstance();
|
|
373
|
+
|
|
374
|
+
// 1. Base de datos transaccional (MySQL)
|
|
375
|
+
const mysql = connector.getEngine(DBSQLType.MySQL, {
|
|
376
|
+
host: 'localhost',
|
|
377
|
+
database: 'ecommerce',
|
|
378
|
+
user: 'root',
|
|
379
|
+
password: 'pass'
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
// 2. Base de datos de documentos (MongoDB)
|
|
383
|
+
const mongo = connector.getEngine(DBNoSQLType.MongoDB, {
|
|
384
|
+
host: 'localhost',
|
|
385
|
+
database: 'ecommerce_content'
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// 3. Base de datos analítica (ClickHouse)
|
|
389
|
+
const clickhouse = connector.getEngine(DBOLAPType.ClickHouse, {
|
|
390
|
+
host: 'http://localhost:8123',
|
|
391
|
+
database: 'analytics'
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
// Procesar orden
|
|
395
|
+
async function processOrder(orderData) {
|
|
396
|
+
const startTime = Date.now();
|
|
397
|
+
|
|
398
|
+
try {
|
|
399
|
+
// 1. Guardar orden en MySQL (transaccional)
|
|
400
|
+
await mysql.withTransaction(async (tx) => {
|
|
401
|
+
const order = await mysql.insert('orders', orderData, { transaction: tx });
|
|
402
|
+
await mysql.update('inventory',
|
|
403
|
+
{ productId: orderData.productId },
|
|
404
|
+
{ stock: { decrement: 1 } },
|
|
405
|
+
{ transaction: tx }
|
|
406
|
+
);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
// 2. Guardar detalles enriquecidos en MongoDB
|
|
410
|
+
await mongo.create(OrderDetailsModel, {
|
|
411
|
+
orderId: orderData.id,
|
|
412
|
+
userPreferences: orderData.preferences,
|
|
413
|
+
metadata: orderData.metadata
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
// 3. Registrar evento en ClickHouse para análisis
|
|
417
|
+
await clickhouse.insert({
|
|
418
|
+
table: 'order_events',
|
|
419
|
+
values: [{
|
|
420
|
+
event_type: 'order_created',
|
|
421
|
+
order_id: orderData.id,
|
|
422
|
+
user_id: orderData.userId,
|
|
423
|
+
amount: orderData.amount,
|
|
424
|
+
duration_ms: Date.now() - startTime,
|
|
425
|
+
timestamp: new Date(),
|
|
426
|
+
success: true
|
|
427
|
+
}]
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
} catch (error) {
|
|
431
|
+
// Registrar error en ClickHouse
|
|
432
|
+
await clickhouse.insert({
|
|
433
|
+
table: 'order_events',
|
|
434
|
+
values: [{
|
|
435
|
+
event_type: 'order_failed',
|
|
436
|
+
order_id: orderData.id,
|
|
437
|
+
error_message: error.message,
|
|
438
|
+
duration_ms: Date.now() - startTime,
|
|
439
|
+
timestamp: new Date(),
|
|
440
|
+
success: false
|
|
441
|
+
}]
|
|
442
|
+
});
|
|
443
|
+
throw error;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
// Análisis de órdenes en tiempo real
|
|
448
|
+
async function getOrderStats() {
|
|
449
|
+
return await clickhouse.query(`
|
|
450
|
+
SELECT
|
|
451
|
+
toStartOfHour(timestamp) as hour,
|
|
452
|
+
countIf(success = true) as successful_orders,
|
|
453
|
+
countIf(success = false) as failed_orders,
|
|
454
|
+
avg(amount) as avg_order_value,
|
|
455
|
+
quantile(0.95)(duration_ms) as p95_duration
|
|
456
|
+
FROM order_events
|
|
457
|
+
WHERE timestamp >= now() - INTERVAL 24 HOUR
|
|
458
|
+
GROUP BY hour
|
|
459
|
+
ORDER BY hour DESC
|
|
460
|
+
`);
|
|
461
|
+
}
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
---
|
|
465
|
+
|
|
93
466
|
## Licencia
|
|
94
467
|
|
|
95
468
|
MIT
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createClient } from '@clickhouse/client';
|
|
1
2
|
import mongoose__default from 'mongoose';
|
|
2
3
|
export * from 'mongoose';
|
|
3
4
|
import { ValidationError, DatabaseError, AccessDeniedError, Sequelize } from 'sequelize';
|
|
@@ -45,6 +46,89 @@ typeof SuppressedError === "function" ? SuppressedError : function (error, suppr
|
|
|
45
46
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
46
47
|
};
|
|
47
48
|
|
|
49
|
+
class OLAPConnector {
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
class ClickHouseConnector extends OLAPConnector {
|
|
53
|
+
constructor(credentials) {
|
|
54
|
+
super();
|
|
55
|
+
this.credentials = credentials;
|
|
56
|
+
let url;
|
|
57
|
+
if (this.credentials.host.match(/^https?:\/\/.+:\d+$/)) {
|
|
58
|
+
url = this.credentials.host;
|
|
59
|
+
}
|
|
60
|
+
else if (this.credentials.host.startsWith('http://') || this.credentials.host.startsWith('https://')) {
|
|
61
|
+
const port = this.credentials.port || 8123;
|
|
62
|
+
url = `${this.credentials.host}:${port}`;
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
const port = this.credentials.port || 8123;
|
|
66
|
+
url = `http://${this.credentials.host}:${port}`;
|
|
67
|
+
}
|
|
68
|
+
this.client = createClient({
|
|
69
|
+
url,
|
|
70
|
+
username: this.credentials.user,
|
|
71
|
+
password: this.credentials.password,
|
|
72
|
+
database: this.credentials.database,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
connect() {
|
|
76
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
77
|
+
const isConnected = yield this.client.ping();
|
|
78
|
+
if (!isConnected) {
|
|
79
|
+
throw new Error('ClickHouse connection failed.');
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
disconnect() {
|
|
84
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
85
|
+
yield this.client.close();
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
query(sql) {
|
|
89
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
90
|
+
const resultSet = yield this.client.query({
|
|
91
|
+
query: sql,
|
|
92
|
+
format: 'JSONEachRow',
|
|
93
|
+
});
|
|
94
|
+
return resultSet.json();
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
insert(options) {
|
|
98
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
99
|
+
const { table, values, format = 'JSONEachRow' } = options;
|
|
100
|
+
yield this.client.insert({
|
|
101
|
+
table,
|
|
102
|
+
values: values,
|
|
103
|
+
format: format,
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
find(table_1) {
|
|
108
|
+
return __awaiter(this, arguments, void 0, function* (table, filters = {}) {
|
|
109
|
+
if (Object.keys(filters).length === 0) {
|
|
110
|
+
const sql = `SELECT * FROM ${table}`;
|
|
111
|
+
return this.query(sql);
|
|
112
|
+
}
|
|
113
|
+
const whereClause = Object.entries(filters)
|
|
114
|
+
.map(([key, value]) => {
|
|
115
|
+
if (typeof value === 'string') {
|
|
116
|
+
return `${key} = '${value}'`;
|
|
117
|
+
}
|
|
118
|
+
else if (typeof value === 'boolean') {
|
|
119
|
+
return `${key} = ${value ? '1' : '0'}`;
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
return `${key} = ${value}`;
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
.join(' AND ');
|
|
126
|
+
const sql = `SELECT * FROM ${table} WHERE ${whereClause}`;
|
|
127
|
+
return this.query(sql);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
48
132
|
class NoSQLConnector {
|
|
49
133
|
constructor() {
|
|
50
134
|
this.config = {};
|
|
@@ -590,6 +674,13 @@ class SQLConnectionManager {
|
|
|
590
674
|
}
|
|
591
675
|
}
|
|
592
676
|
|
|
677
|
+
var DBOLAPType;
|
|
678
|
+
(function (DBOLAPType) {
|
|
679
|
+
DBOLAPType["ClickHouse"] = "ClickHouse";
|
|
680
|
+
DBOLAPType["Druid"] = "Druid";
|
|
681
|
+
DBOLAPType["Snowflake"] = "Snowflake";
|
|
682
|
+
})(DBOLAPType || (DBOLAPType = {}));
|
|
683
|
+
|
|
593
684
|
class DBConnector {
|
|
594
685
|
constructor() {
|
|
595
686
|
this.connection = null;
|
|
@@ -617,6 +708,9 @@ class DBConnector {
|
|
|
617
708
|
else if (Object.values(DBSQLType).includes(type)) {
|
|
618
709
|
this.connection = SQLConnectionManager.getInstance(type, credentials);
|
|
619
710
|
}
|
|
711
|
+
else if (Object.values(DBOLAPType).includes(type)) {
|
|
712
|
+
this.connection = new ClickHouseConnector(credentials);
|
|
713
|
+
}
|
|
620
714
|
else {
|
|
621
715
|
throw new Error(`Unsupported database type: ${type}`);
|
|
622
716
|
}
|
|
@@ -647,6 +741,9 @@ class DBConnector {
|
|
|
647
741
|
else if (Object.values(DBSQLType).includes(type)) {
|
|
648
742
|
this.connection = SQLConnectionManager.getInstance(type, credentials);
|
|
649
743
|
}
|
|
744
|
+
else if (Object.values(DBOLAPType).includes(type)) {
|
|
745
|
+
this.connection = new ClickHouseConnector(credentials);
|
|
746
|
+
}
|
|
650
747
|
else {
|
|
651
748
|
throw new Error(`Unsupported database type: ${type}`);
|
|
652
749
|
}
|
|
@@ -659,4 +756,4 @@ class DBConnector {
|
|
|
659
756
|
}
|
|
660
757
|
DBConnector.instance = null;
|
|
661
758
|
|
|
662
|
-
export { CoreSqlTypes, DBConnector, DBError, DBNoSQLType, DBSQLType, MySQLConnectionError, NoSQLConnector, SqlConnector };
|
|
759
|
+
export { ClickHouseConnector, CoreSqlTypes, DBConnector, DBError, DBNoSQLType, DBOLAPType, DBSQLType, MySQLConnectionError, NoSQLConnector, OLAPConnector, SqlConnector };
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
+
var client = require('@clickhouse/client');
|
|
3
4
|
var mongoose = require('mongoose');
|
|
4
5
|
var sequelize = require('sequelize');
|
|
5
6
|
|
|
@@ -45,6 +46,89 @@ typeof SuppressedError === "function" ? SuppressedError : function (error, suppr
|
|
|
45
46
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
46
47
|
};
|
|
47
48
|
|
|
49
|
+
class OLAPConnector {
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
class ClickHouseConnector extends OLAPConnector {
|
|
53
|
+
constructor(credentials) {
|
|
54
|
+
super();
|
|
55
|
+
this.credentials = credentials;
|
|
56
|
+
let url;
|
|
57
|
+
if (this.credentials.host.match(/^https?:\/\/.+:\d+$/)) {
|
|
58
|
+
url = this.credentials.host;
|
|
59
|
+
}
|
|
60
|
+
else if (this.credentials.host.startsWith('http://') || this.credentials.host.startsWith('https://')) {
|
|
61
|
+
const port = this.credentials.port || 8123;
|
|
62
|
+
url = `${this.credentials.host}:${port}`;
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
const port = this.credentials.port || 8123;
|
|
66
|
+
url = `http://${this.credentials.host}:${port}`;
|
|
67
|
+
}
|
|
68
|
+
this.client = client.createClient({
|
|
69
|
+
url,
|
|
70
|
+
username: this.credentials.user,
|
|
71
|
+
password: this.credentials.password,
|
|
72
|
+
database: this.credentials.database,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
connect() {
|
|
76
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
77
|
+
const isConnected = yield this.client.ping();
|
|
78
|
+
if (!isConnected) {
|
|
79
|
+
throw new Error('ClickHouse connection failed.');
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
disconnect() {
|
|
84
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
85
|
+
yield this.client.close();
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
query(sql) {
|
|
89
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
90
|
+
const resultSet = yield this.client.query({
|
|
91
|
+
query: sql,
|
|
92
|
+
format: 'JSONEachRow',
|
|
93
|
+
});
|
|
94
|
+
return resultSet.json();
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
insert(options) {
|
|
98
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
99
|
+
const { table, values, format = 'JSONEachRow' } = options;
|
|
100
|
+
yield this.client.insert({
|
|
101
|
+
table,
|
|
102
|
+
values: values,
|
|
103
|
+
format: format,
|
|
104
|
+
});
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
find(table_1) {
|
|
108
|
+
return __awaiter(this, arguments, void 0, function* (table, filters = {}) {
|
|
109
|
+
if (Object.keys(filters).length === 0) {
|
|
110
|
+
const sql = `SELECT * FROM ${table}`;
|
|
111
|
+
return this.query(sql);
|
|
112
|
+
}
|
|
113
|
+
const whereClause = Object.entries(filters)
|
|
114
|
+
.map(([key, value]) => {
|
|
115
|
+
if (typeof value === 'string') {
|
|
116
|
+
return `${key} = '${value}'`;
|
|
117
|
+
}
|
|
118
|
+
else if (typeof value === 'boolean') {
|
|
119
|
+
return `${key} = ${value ? '1' : '0'}`;
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
return `${key} = ${value}`;
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
.join(' AND ');
|
|
126
|
+
const sql = `SELECT * FROM ${table} WHERE ${whereClause}`;
|
|
127
|
+
return this.query(sql);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
48
132
|
class NoSQLConnector {
|
|
49
133
|
constructor() {
|
|
50
134
|
this.config = {};
|
|
@@ -590,6 +674,13 @@ class SQLConnectionManager {
|
|
|
590
674
|
}
|
|
591
675
|
}
|
|
592
676
|
|
|
677
|
+
exports.DBOLAPType = void 0;
|
|
678
|
+
(function (DBOLAPType) {
|
|
679
|
+
DBOLAPType["ClickHouse"] = "ClickHouse";
|
|
680
|
+
DBOLAPType["Druid"] = "Druid";
|
|
681
|
+
DBOLAPType["Snowflake"] = "Snowflake";
|
|
682
|
+
})(exports.DBOLAPType || (exports.DBOLAPType = {}));
|
|
683
|
+
|
|
593
684
|
class DBConnector {
|
|
594
685
|
constructor() {
|
|
595
686
|
this.connection = null;
|
|
@@ -617,6 +708,9 @@ class DBConnector {
|
|
|
617
708
|
else if (Object.values(exports.DBSQLType).includes(type)) {
|
|
618
709
|
this.connection = SQLConnectionManager.getInstance(type, credentials);
|
|
619
710
|
}
|
|
711
|
+
else if (Object.values(exports.DBOLAPType).includes(type)) {
|
|
712
|
+
this.connection = new ClickHouseConnector(credentials);
|
|
713
|
+
}
|
|
620
714
|
else {
|
|
621
715
|
throw new Error(`Unsupported database type: ${type}`);
|
|
622
716
|
}
|
|
@@ -647,6 +741,9 @@ class DBConnector {
|
|
|
647
741
|
else if (Object.values(exports.DBSQLType).includes(type)) {
|
|
648
742
|
this.connection = SQLConnectionManager.getInstance(type, credentials);
|
|
649
743
|
}
|
|
744
|
+
else if (Object.values(exports.DBOLAPType).includes(type)) {
|
|
745
|
+
this.connection = new ClickHouseConnector(credentials);
|
|
746
|
+
}
|
|
650
747
|
else {
|
|
651
748
|
throw new Error(`Unsupported database type: ${type}`);
|
|
652
749
|
}
|
|
@@ -687,10 +784,12 @@ Object.defineProperty(exports, "fn", {
|
|
|
687
784
|
enumerable: true,
|
|
688
785
|
get: function () { return sequelize.fn; }
|
|
689
786
|
});
|
|
787
|
+
exports.ClickHouseConnector = ClickHouseConnector;
|
|
690
788
|
exports.DBConnector = DBConnector;
|
|
691
789
|
exports.DBError = DBError;
|
|
692
790
|
exports.MySQLConnectionError = MySQLConnectionError;
|
|
693
791
|
exports.NoSQLConnector = NoSQLConnector;
|
|
792
|
+
exports.OLAPConnector = OLAPConnector;
|
|
694
793
|
exports.SqlConnector = SqlConnector;
|
|
695
794
|
Object.keys(mongoose).forEach(function (k) {
|
|
696
795
|
if (k !== 'default' && !Object.prototype.hasOwnProperty.call(exports, k)) Object.defineProperty(exports, k, {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { CredentialsConnection } from '../types/CredentialsConnection';
|
|
2
|
+
import { OLAPConnector, InsertOptions, ClickHouseInsertFormat } from '../abstract/OLAPConnector';
|
|
3
|
+
export declare class ClickHouseConnector extends OLAPConnector {
|
|
4
|
+
private client;
|
|
5
|
+
private credentials;
|
|
6
|
+
constructor(credentials: CredentialsConnection);
|
|
7
|
+
connect(): Promise<void>;
|
|
8
|
+
disconnect(): Promise<void>;
|
|
9
|
+
query<T = unknown>(sql: string): Promise<T[]>;
|
|
10
|
+
insert<F extends ClickHouseInsertFormat = 'JSONEachRow'>(options: InsertOptions<F>): Promise<void>;
|
|
11
|
+
find<T = unknown>(table: string, filters?: Record<string, any>): Promise<T[]>;
|
|
12
|
+
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { NoSQLConnector } from "./abstract/NoSQLConnector";
|
|
2
|
+
import { OLAPConnector } from "./abstract/OLAPConnector";
|
|
2
3
|
import { SqlConnector } from "./abstract/SQLConnector";
|
|
3
4
|
import { CredentialsConnection } from "./types/CredentialsConnection";
|
|
4
5
|
import { DBNoSQLType } from "./types/DBNoSQLTypes";
|
|
5
6
|
import { DBSQLType } from "./types/DBSQLTypes";
|
|
7
|
+
import { DBOLAPType } from "./types/OLAPTypes";
|
|
6
8
|
export declare class DBConnector {
|
|
7
9
|
private static instance;
|
|
8
10
|
private connection;
|
|
@@ -11,8 +13,8 @@ export declare class DBConnector {
|
|
|
11
13
|
private dbType;
|
|
12
14
|
constructor();
|
|
13
15
|
static getInstance(): DBConnector;
|
|
14
|
-
createConnection<T extends SqlConnector | NoSQLConnector>(type: DBSQLType | DBNoSQLType, credentials: CredentialsConnection): Promise<T>;
|
|
15
|
-
getConnection<T extends NoSQLConnector | SqlConnector>(): T | null;
|
|
16
|
+
createConnection<T extends SqlConnector | NoSQLConnector | OLAPConnector>(type: DBSQLType | DBNoSQLType | DBOLAPType, credentials: CredentialsConnection): Promise<T>;
|
|
17
|
+
getConnection<T extends NoSQLConnector | SqlConnector | OLAPConnector>(): T | null;
|
|
16
18
|
closeConnection(): Promise<void>;
|
|
17
|
-
getEngine<T extends NoSQLConnector | SqlConnector>(type: DBNoSQLType | DBSQLType, credentials: CredentialsConnection): T;
|
|
19
|
+
getEngine<T extends NoSQLConnector | SqlConnector | OLAPConnector>(type: DBNoSQLType | DBSQLType | DBOLAPType, credentials: CredentialsConnection): T;
|
|
18
20
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type ClickHouseInsertFormat = 'JSONEachRow' | 'JSON' | 'JSONCompactEachRow' | 'CSV' | 'TabSeparated' | 'TabSeparatedWithNames' | 'Values';
|
|
2
|
+
export type ClickHouseInsertValues<T> = T extends 'JSONEachRow' | 'JSONCompactEachRow' | 'JSON' ? Record<string, any>[] : string;
|
|
3
|
+
export interface InsertOptions<F extends ClickHouseInsertFormat = 'JSONEachRow'> {
|
|
4
|
+
table: string;
|
|
5
|
+
values: ClickHouseInsertValues<F>;
|
|
6
|
+
format?: F;
|
|
7
|
+
}
|
|
8
|
+
export declare abstract class OLAPConnector {
|
|
9
|
+
abstract connect(): Promise<void>;
|
|
10
|
+
abstract disconnect(): Promise<void>;
|
|
11
|
+
abstract query<T = unknown>(sql: string): Promise<T[]>;
|
|
12
|
+
abstract insert<F extends ClickHouseInsertFormat = 'JSONEachRow'>(options: InsertOptions<F>): Promise<void>;
|
|
13
|
+
abstract find<T = unknown>(table: string, filters: Record<string, any>): Promise<T[]>;
|
|
14
|
+
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export { Recorset } from './types/RecordSet';
|
|
|
4
4
|
export { DBConnector } from "./DBConnector";
|
|
5
5
|
export { DBSQLType } from "./types/DBSQLTypes";
|
|
6
6
|
export { DBNoSQLType } from "./types/DBNoSQLTypes";
|
|
7
|
+
export { DBOLAPType } from "./types/OLAPTypes";
|
|
7
8
|
export { NoSQLConnector } from "./abstract/NoSQLConnector";
|
|
8
9
|
export { SqlConnector, Filter } from "./abstract/SQLConnector";
|
|
9
10
|
export { NoSQLOptions } from "./types/NoSQLOptions";
|
|
@@ -11,3 +12,5 @@ export * from 'mongoose';
|
|
|
11
12
|
export { Model as ModelSequelize, Sequelize, DataTypes, CreateOptions, UpdateOptions, DestroyOptions, FindOptions, Attributes, WhereOptions, QueryOptions, InferAttributes, CreationAttributes, InferCreationAttributes, QueryTypes, QueryOptionsWithType, Transaction, Op, fn } from "sequelize";
|
|
12
13
|
export { DBError } from "./errors/DBError";
|
|
13
14
|
export { MySQLConnectionError } from "./errors/MySQLConnectionError";
|
|
15
|
+
export { OLAPConnector, InsertOptions, ClickHouseInsertFormat, ClickHouseInsertValues } from './abstract/OLAPConnector';
|
|
16
|
+
export { ClickHouseConnector } from './Core/ClickHouseConnector';
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { OLAPConnector } from "../abstract/OLAPConnector";
|
|
2
|
+
import { CredentialsConnection } from "../types/CredentialsConnection";
|
|
3
|
+
import { DBOLAPType } from "../types/OLAPTypes";
|
|
4
|
+
export declare class OLAPConnectionManager {
|
|
5
|
+
static getInstance(type: DBOLAPType, credentials: CredentialsConnection): OLAPConnector;
|
|
6
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hemia/db-connector",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.6",
|
|
4
4
|
"description": "Hemia Database Conector",
|
|
5
5
|
"main": "dist/hemia-db-connector.js",
|
|
6
6
|
"module": "dist/hemia-db-connector.esm.js",
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
"author": "Hemia Technologies",
|
|
41
41
|
"license": "MIT",
|
|
42
42
|
"dependencies": {
|
|
43
|
+
"@clickhouse/client": "^1.14.0",
|
|
43
44
|
"eslint": "^9.21.0",
|
|
44
45
|
"mongoose": "^8.8.2"
|
|
45
46
|
},
|
|
@@ -48,15 +49,14 @@
|
|
|
48
49
|
"@rollup/plugin-json": "^6.1.0",
|
|
49
50
|
"@rollup/plugin-node-resolve": "^15.2.3",
|
|
50
51
|
"@types/chai": "^4.3.17",
|
|
52
|
+
"@types/jest": "^29.5.14",
|
|
51
53
|
"@types/mocha": "^10.0.7",
|
|
52
54
|
"@types/mssql": "^9.1.5",
|
|
53
55
|
"@types/node": "^22.3.0",
|
|
54
56
|
"@typescript-eslint/eslint-plugin": "^8.5.0",
|
|
55
|
-
"@types/jest": "^29.5.14",
|
|
56
57
|
"chai": "^4.2.0",
|
|
57
58
|
"events": "^3.3.0",
|
|
58
59
|
"jest": "^29.7.0",
|
|
59
|
-
"ts-jest": "^29.2.5",
|
|
60
60
|
"mocha": "^10.7.3",
|
|
61
61
|
"mssql": "^11.0.1",
|
|
62
62
|
"mysql2": "^3.11.0",
|
|
@@ -64,6 +64,7 @@
|
|
|
64
64
|
"rollup": "^4.20.0",
|
|
65
65
|
"rollup-plugin-typescript2": "^0.36.0",
|
|
66
66
|
"sequelize": "^6.37.3",
|
|
67
|
+
"ts-jest": "^29.2.5",
|
|
67
68
|
"ts-node": "^8.9.0",
|
|
68
69
|
"typescript": "^5.5.4"
|
|
69
70
|
},
|