@hypequery/clickhouse 1.4.0-beta.2 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -2
- package/dist/cli/bin.js +65 -23
- package/dist/cli/generate-types.js +41 -3
- package/dist/core/cache/cache-manager.d.ts +4 -0
- package/dist/core/cache/cache-manager.d.ts.map +1 -0
- package/dist/core/cache/cache-manager.js +176 -0
- package/dist/core/cache/controller.d.ts +15 -0
- package/dist/core/cache/controller.d.ts.map +1 -0
- package/dist/core/cache/controller.js +58 -0
- package/dist/core/cache/key.d.ts +11 -0
- package/dist/core/cache/key.d.ts.map +1 -0
- package/dist/core/cache/key.js +26 -0
- package/dist/core/cache/providers/memory-lru.d.ts +31 -0
- package/dist/core/cache/providers/memory-lru.d.ts.map +1 -0
- package/dist/core/cache/providers/memory-lru.js +156 -0
- package/dist/core/cache/providers/noop.d.ts +7 -0
- package/dist/core/cache/providers/noop.d.ts.map +1 -0
- package/dist/core/cache/providers/noop.js +11 -0
- package/dist/core/cache/runtime-context.d.ts +30 -0
- package/dist/core/cache/runtime-context.d.ts.map +1 -0
- package/dist/core/cache/runtime-context.js +58 -0
- package/dist/core/cache/serialization.d.ts +6 -0
- package/dist/core/cache/serialization.d.ts.map +1 -0
- package/dist/core/cache/serialization.js +166 -0
- package/dist/core/cache/types.d.ts +52 -0
- package/dist/core/cache/types.d.ts.map +1 -0
- package/dist/core/cache/types.js +1 -0
- package/dist/core/cache/utils.d.ts +9 -0
- package/dist/core/cache/utils.d.ts.map +1 -0
- package/dist/core/cache/utils.js +30 -0
- package/dist/core/connection.d.ts.map +1 -1
- package/dist/core/connection.js +4 -3
- package/dist/core/cross-filter.d.ts +12 -9
- package/dist/core/cross-filter.d.ts.map +1 -1
- package/dist/core/cross-filter.js +9 -6
- package/dist/core/env/auto-client.browser.d.ts +3 -0
- package/dist/core/env/auto-client.browser.d.ts.map +1 -0
- package/dist/core/env/auto-client.browser.js +3 -0
- package/dist/core/env/auto-client.d.ts +9 -0
- package/dist/core/env/auto-client.d.ts.map +1 -0
- package/dist/core/env/auto-client.js +21 -0
- package/dist/core/features/aggregations.d.ts +18 -22
- package/dist/core/features/aggregations.d.ts.map +1 -1
- package/dist/core/features/aggregations.js +6 -6
- package/dist/core/features/analytics.d.ts +15 -19
- package/dist/core/features/analytics.d.ts.map +1 -1
- package/dist/core/features/analytics.js +2 -2
- package/dist/core/features/cross-filtering.d.ts +4 -24
- package/dist/core/features/cross-filtering.d.ts.map +1 -1
- package/dist/core/features/cross-filtering.js +0 -34
- package/dist/core/features/executor.d.ts +11 -9
- package/dist/core/features/executor.d.ts.map +1 -1
- package/dist/core/features/executor.js +14 -5
- package/dist/core/features/filtering.d.ts +32 -28
- package/dist/core/features/filtering.d.ts.map +1 -1
- package/dist/core/features/filtering.js +27 -26
- package/dist/core/features/joins.d.ts +7 -10
- package/dist/core/features/joins.d.ts.map +1 -1
- package/dist/core/features/pagination.d.ts +8 -10
- package/dist/core/features/pagination.d.ts.map +1 -1
- package/dist/core/features/pagination.js +15 -42
- package/dist/core/features/query-modifiers.d.ts +18 -21
- package/dist/core/features/query-modifiers.d.ts.map +1 -1
- package/dist/core/formatters/sql-formatter.d.ts.map +1 -1
- package/dist/core/formatters/sql-formatter.js +6 -0
- package/dist/core/join-relationships.d.ts +2 -1
- package/dist/core/join-relationships.d.ts.map +1 -1
- package/dist/core/query-builder.d.ts +69 -74
- package/dist/core/query-builder.d.ts.map +1 -1
- package/dist/core/query-builder.js +155 -114
- package/dist/core/tests/integration/pagination-test-tbc.js +1 -0
- package/dist/core/tests/integration/setup.d.ts +8 -1
- package/dist/core/tests/integration/setup.d.ts.map +1 -1
- package/dist/core/tests/integration/setup.js +55 -22
- package/dist/core/tests/integration/test-config.d.ts +2 -2
- package/dist/core/tests/integration/test-config.d.ts.map +1 -1
- package/dist/core/tests/integration/test-config.js +3 -4
- package/dist/core/tests/integration/test-data.json +190 -0
- package/dist/core/tests/test-utils.d.ts +18 -3
- package/dist/core/tests/test-utils.d.ts.map +1 -1
- package/dist/core/tests/test-utils.js +37 -10
- package/dist/core/types/builder-state.d.ts +25 -0
- package/dist/core/types/builder-state.d.ts.map +1 -0
- package/dist/core/types/builder-state.js +1 -0
- package/dist/core/types/select-types.d.ts +32 -0
- package/dist/core/types/select-types.d.ts.map +1 -0
- package/dist/core/types/select-types.js +1 -0
- package/dist/core/types/type-helpers.d.ts +5 -0
- package/dist/core/types/type-helpers.d.ts.map +1 -0
- package/dist/core/types/type-helpers.js +1 -0
- package/dist/core/utils/logger.d.ts +6 -0
- package/dist/core/utils/logger.d.ts.map +1 -1
- package/dist/core/utils/logger.js +7 -2
- package/dist/core/utils/predicate-builder.d.ts +29 -0
- package/dist/core/utils/predicate-builder.d.ts.map +1 -0
- package/dist/core/utils/predicate-builder.js +92 -0
- package/dist/core/utils/sql-expressions.d.ts +24 -10
- package/dist/core/utils/sql-expressions.d.ts.map +1 -1
- package/dist/core/utils/sql-expressions.js +7 -30
- package/dist/core/utils/streaming-helpers.d.ts +2 -0
- package/dist/core/utils/streaming-helpers.d.ts.map +1 -0
- package/dist/core/utils/streaming-helpers.js +137 -0
- package/dist/core/validators/filter-validator.d.ts +2 -1
- package/dist/core/validators/filter-validator.d.ts.map +1 -1
- package/dist/core/validators/value-validator.d.ts +2 -1
- package/dist/core/validators/value-validator.d.ts.map +1 -1
- package/dist/index.d.ts +11 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/types/base.d.ts +10 -15
- package/dist/types/base.d.ts.map +1 -1
- package/dist/types/clickhouse-types.d.ts +9 -4
- package/dist/types/clickhouse-types.d.ts.map +1 -1
- package/dist/types/filters.d.ts +1 -1
- package/dist/types/filters.d.ts.map +1 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +2 -0
- package/dist/types/schema.d.ts +19 -0
- package/dist/types/schema.d.ts.map +1 -0
- package/dist/types/schema.js +1 -0
- package/package.json +14 -12
- package/dist/core/tests/integration/test-initializer.d.ts +0 -7
- package/dist/core/tests/integration/test-initializer.d.ts.map +0 -1
- package/dist/core/tests/integration/test-initializer.js +0 -32
package/README.md
CHANGED
|
@@ -95,6 +95,39 @@ const results = await db
|
|
|
95
95
|
.execute();
|
|
96
96
|
```
|
|
97
97
|
|
|
98
|
+
## Caching (Experimental)
|
|
99
|
+
|
|
100
|
+
The query builder can cache results with deterministic keys, stale-while-revalidate behavior, and pluggable providers.
|
|
101
|
+
|
|
102
|
+
```typescript
|
|
103
|
+
import { createQueryBuilder, MemoryCacheProvider } from '@hypequery/clickhouse';
|
|
104
|
+
|
|
105
|
+
const db = createQueryBuilder<IntrospectedSchema>({
|
|
106
|
+
host: 'your-clickhouse-host',
|
|
107
|
+
username: 'default',
|
|
108
|
+
password: '',
|
|
109
|
+
database: 'default',
|
|
110
|
+
cache: {
|
|
111
|
+
mode: 'stale-while-revalidate',
|
|
112
|
+
ttlMs: 2_000,
|
|
113
|
+
staleTtlMs: 30_000,
|
|
114
|
+
provider: new MemoryCacheProvider({ maxEntries: 5_000 })
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
const rows = await db
|
|
119
|
+
.table('users')
|
|
120
|
+
.select(['id', 'email'])
|
|
121
|
+
.where('active', '=', true)
|
|
122
|
+
.cache({ tags: ['users'] })
|
|
123
|
+
.execute();
|
|
124
|
+
|
|
125
|
+
// Programmatic invalidation
|
|
126
|
+
await db.cache.invalidateTags(['users']);
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Use `.cache()` to attach defaults to a fluent chain, pass `execute({ cache: { ... } })` for one-off overrides, or call `db.cache.*` for manual invalidation. For a deep dive on cache modes, invalidation, advanced serialization, and bring-your-own-provider recipes (Redis/Upstash, compression, etc.), see the [Caching guide](https://hypequery.com/docs/features/caching).
|
|
130
|
+
|
|
98
131
|
## Schema Generation
|
|
99
132
|
|
|
100
133
|
hypequery provides a CLI tool to generate TypeScript types from your ClickHouse schema:
|
|
@@ -184,8 +217,25 @@ const tripsWithDrivers = await db.table('trips')
|
|
|
184
217
|
.join('drivers', 'trips.driver_id', 'drivers.id')
|
|
185
218
|
.execute();
|
|
186
219
|
|
|
220
|
+
// After joining, TypeScript understands the expanded scope
|
|
221
|
+
const tripsWithUsers = await db.table('trips')
|
|
222
|
+
.innerJoin('users', 'trips.user_id', 'users.id')
|
|
223
|
+
.select(['users.email', 'trips.trip_id'])
|
|
224
|
+
.where('users.email', 'like', '%@example.com')
|
|
225
|
+
.execute();
|
|
226
|
+
|
|
227
|
+
// Keep literal column inference with selectConst and reuse joined columns in ORDER BY / HAVING
|
|
228
|
+
const sortedTrips = await db.table('trips')
|
|
229
|
+
.innerJoin('users', 'trips.user_id', 'users.id')
|
|
230
|
+
.selectConst('users.email', 'trips.trip_id')
|
|
231
|
+
.groupBy(['users.email', 'trips.trip_id'])
|
|
232
|
+
.having('COUNT(*) > 1')
|
|
233
|
+
.orderBy('users.email', 'DESC')
|
|
234
|
+
.execute();
|
|
235
|
+
|
|
187
236
|
```
|
|
188
237
|
|
|
238
|
+
`selectConst()` preserves literal column names (including aliases like `users.email`), which means TypeScript keeps those identifiers available for downstream `orderBy`, `groupBy`, and `having` calls.
|
|
189
239
|
|
|
190
240
|
**Benefits:**
|
|
191
241
|
- ✅ Works in all environments (Node.js, browser, bundlers)
|
|
@@ -205,6 +255,20 @@ const db = createQueryBuilder<IntrospectedSchema>({
|
|
|
205
255
|
```
|
|
206
256
|
|
|
207
257
|
|
|
258
|
+
## Testing
|
|
259
|
+
|
|
260
|
+
Run the fast feedback loop with:
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
npm run test
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
This command runs type checks + unit tests only. To exercise the ClickHouse-backed integration suite, ensure a test ClickHouse instance is available (or set the required env vars) and run:
|
|
267
|
+
|
|
268
|
+
```bash
|
|
269
|
+
npm run test:integration
|
|
270
|
+
```
|
|
271
|
+
|
|
208
272
|
## Versioning and Release Channels
|
|
209
273
|
|
|
210
274
|
hypequery follows semantic versioning and provides multiple release channels:
|
|
@@ -219,7 +283,6 @@ For detailed documentation and examples, visit our [documentation site](https://
|
|
|
219
283
|
- [Getting Started](https://hypequery.com/docs/installation)
|
|
220
284
|
- [Query Building](https://hypequery.com/docs/guides/query-building)
|
|
221
285
|
- [Filtering](https://hypequery.com/docs/guides/filtering)
|
|
222
|
-
- [Pagination](https://hypequery.com/docs/features/pagination)
|
|
223
286
|
- [API Reference](https://hypequery.com/docs/reference/api)
|
|
224
287
|
|
|
225
288
|
|
|
@@ -250,4 +313,4 @@ This project is licensed under the Apache-2.0 License - see the [LICENSE](LICENS
|
|
|
250
313
|
|
|
251
314
|
<div align="center">
|
|
252
315
|
<sub>Built with ❤️ by the hypequery team</sub>
|
|
253
|
-
</div>
|
|
316
|
+
</div>
|
package/dist/cli/bin.js
CHANGED
|
@@ -50,6 +50,8 @@ ${colors.bright}Options:${colors.reset}
|
|
|
50
50
|
--secure Use HTTPS/TLS for connection
|
|
51
51
|
--help, -h Show this help text
|
|
52
52
|
|
|
53
|
+
${colors.dim}Note: All options support both formats: --option=value or --option value${colors.reset}
|
|
54
|
+
|
|
53
55
|
${colors.bright}Environment variables:${colors.reset}
|
|
54
56
|
CLICKHOUSE_HOST ClickHouse server URL
|
|
55
57
|
VITE_CLICKHOUSE_HOST Alternative variable for Vite projects
|
|
@@ -70,9 +72,11 @@ ${colors.bright}Environment variables:${colors.reset}
|
|
|
70
72
|
${colors.bright}Examples:${colors.reset}
|
|
71
73
|
npx hypequery-generate-types
|
|
72
74
|
npx hypequery-generate-types --output=./src/types/db-schema.ts
|
|
75
|
+
npx hypequery-generate-types --output ./src/types/db-schema.ts
|
|
73
76
|
npx hypequery-generate-types --host=https://your-instance.clickhouse.cloud:8443 --secure
|
|
74
|
-
npx hypequery-generate-types --host
|
|
77
|
+
npx hypequery-generate-types --host http://localhost:8123 --username default --password password --database my_db
|
|
75
78
|
npx hypequery-generate-types --include-tables=users,orders,products
|
|
79
|
+
npx hypequery-generate-types --include-tables users,orders,products
|
|
76
80
|
`);
|
|
77
81
|
}
|
|
78
82
|
|
|
@@ -87,28 +91,66 @@ function parseArguments(args) {
|
|
|
87
91
|
secure: false
|
|
88
92
|
};
|
|
89
93
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
94
|
+
// Helper function to extract value from --param=value format
|
|
95
|
+
function getParamValue(arg, paramName) {
|
|
96
|
+
if (arg.startsWith(`${paramName}=`)) {
|
|
97
|
+
return arg.substring(paramName.length + 1); // +1 for the '=' character
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Helper function to get next argument as value
|
|
103
|
+
function getNextArgValue(args, index) {
|
|
104
|
+
return args[index + 1];
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Parameter handlers map
|
|
108
|
+
const paramHandlers = {
|
|
109
|
+
'--output': (value) => config.output = value,
|
|
110
|
+
'--host': (value) => config.host = value,
|
|
111
|
+
'--username': (value) => config.username = value,
|
|
112
|
+
'--password': (value) => config.password = value,
|
|
113
|
+
'--database': (value) => config.database = value,
|
|
114
|
+
'--include-tables': (value) => config.includeTables = value.split(','),
|
|
115
|
+
'--exclude-tables': (value) => config.excludeTables = value.split(',')
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
for (let i = 0; i < args.length; i++) {
|
|
119
|
+
const arg = args[i];
|
|
120
|
+
let handled = false;
|
|
121
|
+
|
|
122
|
+
// Handle parameters with values
|
|
123
|
+
for (const [paramName, handler] of Object.entries(paramHandlers)) {
|
|
124
|
+
// Check for --param=value format
|
|
125
|
+
const value = getParamValue(arg, paramName);
|
|
126
|
+
if (value !== null) {
|
|
127
|
+
handler(value);
|
|
128
|
+
handled = true;
|
|
129
|
+
break;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Check for --param value format
|
|
133
|
+
if (arg === paramName) {
|
|
134
|
+
const nextValue = getNextArgValue(args, i);
|
|
135
|
+
if (nextValue && !nextValue.startsWith('-')) {
|
|
136
|
+
handler(nextValue);
|
|
137
|
+
i++; // Skip the next argument since we consumed it
|
|
138
|
+
handled = true;
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Handle boolean flags
|
|
145
|
+
if (!handled) {
|
|
146
|
+
if (arg === '--secure') {
|
|
147
|
+
config.secure = true;
|
|
148
|
+
} else if (arg === '--help' || arg === '-h') {
|
|
149
|
+
config.showHelp = true;
|
|
150
|
+
} else if (!arg.startsWith('-') && !config.output) {
|
|
151
|
+
// For backwards compatibility, treat the first non-flag argument as the output path
|
|
152
|
+
config.output = arg;
|
|
153
|
+
}
|
|
112
154
|
}
|
|
113
155
|
}
|
|
114
156
|
|
|
@@ -35,6 +35,40 @@ const clickhouseToTsType = (type) => {
|
|
|
35
35
|
return `${clickhouseToTsType(innerType)} | null`;
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
// Handle Map types
|
|
39
|
+
if (type.startsWith('Map(')) {
|
|
40
|
+
// Extract key and value types from Map(KeyType, ValueType)
|
|
41
|
+
const mapContent = type.slice(4, -1); // Remove 'Map(' and ')'
|
|
42
|
+
const commaIndex = mapContent.lastIndexOf(',');
|
|
43
|
+
if (commaIndex !== -1) {
|
|
44
|
+
const keyType = mapContent.substring(0, commaIndex).trim();
|
|
45
|
+
const valueType = mapContent.substring(commaIndex + 1).trim();
|
|
46
|
+
|
|
47
|
+
// Handle different key types
|
|
48
|
+
let keyTsType = 'string';
|
|
49
|
+
if (keyType === 'LowCardinality(String)') {
|
|
50
|
+
keyTsType = 'string';
|
|
51
|
+
} else if (keyType.includes('Int') || keyType.includes('UInt')) {
|
|
52
|
+
keyTsType = 'number';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Handle different value types
|
|
56
|
+
let valueTsType = 'unknown';
|
|
57
|
+
if (valueType.startsWith('Array(')) {
|
|
58
|
+
const innerType = valueType.slice(6, -1);
|
|
59
|
+
valueTsType = `Array<${clickhouseToTsType(innerType)}>`;
|
|
60
|
+
} else if (valueType.startsWith('Nullable(')) {
|
|
61
|
+
const innerType = valueType.slice(9, -1);
|
|
62
|
+
valueTsType = `${clickhouseToTsType(innerType)} | null`;
|
|
63
|
+
} else {
|
|
64
|
+
valueTsType = clickhouseToTsType(valueType);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return `Record<${keyTsType}, ${valueTsType}>`;
|
|
68
|
+
}
|
|
69
|
+
return 'Record<string, unknown>';
|
|
70
|
+
}
|
|
71
|
+
|
|
38
72
|
switch (type.toLowerCase()) {
|
|
39
73
|
case 'string':
|
|
40
74
|
case 'fixedstring':
|
|
@@ -43,12 +77,16 @@ const clickhouseToTsType = (type) => {
|
|
|
43
77
|
case 'int16':
|
|
44
78
|
case 'int32':
|
|
45
79
|
case 'uint8':
|
|
80
|
+
case 'int64':
|
|
46
81
|
case 'uint16':
|
|
47
82
|
case 'uint32':
|
|
48
|
-
return 'number';
|
|
49
|
-
case 'int64':
|
|
50
83
|
case 'uint64':
|
|
51
|
-
return '
|
|
84
|
+
return 'number';
|
|
85
|
+
case 'uint128':
|
|
86
|
+
case 'uint256':
|
|
87
|
+
case 'int128':
|
|
88
|
+
case 'int256':
|
|
89
|
+
return 'string';
|
|
52
90
|
case 'float32':
|
|
53
91
|
case 'float64':
|
|
54
92
|
case 'decimal':
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { QueryBuilder, ExecuteOptions } from '../query-builder.js';
|
|
2
|
+
import type { AnyBuilderState, SchemaDefinition } from '../types/builder-state.js';
|
|
3
|
+
export declare function executeWithCache<Schema extends SchemaDefinition<Schema>, State extends AnyBuilderState>(builder: QueryBuilder<Schema, State>, options?: ExecuteOptions): Promise<State['output'][]>;
|
|
4
|
+
//# sourceMappingURL=cache-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache-manager.d.ts","sourceRoot":"","sources":["../../../src/core/cache/cache-manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,KAAK,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AA6DnF,wBAAsB,gBAAgB,CACpC,MAAM,SAAS,gBAAgB,CAAC,MAAM,CAAC,EACvC,KAAK,SAAS,eAAe,EAE7B,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,EACpC,OAAO,CAAC,EAAE,cAAc,GACvB,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CA0J5B"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { computeCacheKey } from './key.js';
|
|
2
|
+
import { mergeCacheOptions } from './runtime-context.js';
|
|
3
|
+
import { logger } from '../utils/logger.js';
|
|
4
|
+
import { substituteParameters } from '../utils.js';
|
|
5
|
+
function isCacheable(options) {
|
|
6
|
+
const ttl = options.ttlMs ?? 0;
|
|
7
|
+
const stale = options.staleTtlMs ?? 0;
|
|
8
|
+
return ttl > 0 || stale > 0;
|
|
9
|
+
}
|
|
10
|
+
function deriveTags(builder) {
|
|
11
|
+
const tags = new Set();
|
|
12
|
+
tags.add(builder.getTableName());
|
|
13
|
+
const joins = builder.getConfig().joins || [];
|
|
14
|
+
joins.forEach(join => tags.add(join.table));
|
|
15
|
+
return Array.from(tags);
|
|
16
|
+
}
|
|
17
|
+
async function logCacheHit({ sql, parameters, status, cacheKey, options, rowCount, ageMs, queryId }) {
|
|
18
|
+
const finalSQL = substituteParameters(sql, parameters);
|
|
19
|
+
const timestamp = Date.now();
|
|
20
|
+
logger.logQuery({
|
|
21
|
+
query: finalSQL,
|
|
22
|
+
parameters,
|
|
23
|
+
startTime: timestamp,
|
|
24
|
+
endTime: timestamp,
|
|
25
|
+
duration: 0,
|
|
26
|
+
status: 'completed',
|
|
27
|
+
rowCount,
|
|
28
|
+
queryId,
|
|
29
|
+
cacheStatus: status,
|
|
30
|
+
cacheKey,
|
|
31
|
+
cacheMode: options.mode,
|
|
32
|
+
cacheAgeMs: ageMs,
|
|
33
|
+
cacheRowCount: rowCount
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
export async function executeWithCache(builder, options) {
|
|
37
|
+
const runtime = builder.getRuntimeContext();
|
|
38
|
+
const provider = runtime.provider;
|
|
39
|
+
const normalizedExecuteCache = options?.cache === false
|
|
40
|
+
? { mode: 'no-store' }
|
|
41
|
+
: options?.cache;
|
|
42
|
+
const mergedOptions = mergeCacheOptions(runtime.defaults, builder.getCacheOptions(), normalizedExecuteCache);
|
|
43
|
+
const mode = mergedOptions.mode ?? 'no-store';
|
|
44
|
+
if (!provider || mode === 'no-store' || !isCacheable(mergedOptions)) {
|
|
45
|
+
return runWithoutCache('bypass');
|
|
46
|
+
}
|
|
47
|
+
const activeProvider = provider;
|
|
48
|
+
const { sql, parameters } = builder.toSQLWithParams();
|
|
49
|
+
const tableName = builder.getTableName();
|
|
50
|
+
const namespace = mergedOptions.namespace || runtime.namespace;
|
|
51
|
+
const key = mergedOptions.key || computeCacheKey({
|
|
52
|
+
namespace,
|
|
53
|
+
sql,
|
|
54
|
+
parameters,
|
|
55
|
+
settings: builder.getConfig().settings ? { settings: builder.getConfig().settings } : undefined,
|
|
56
|
+
version: runtime.versionTag,
|
|
57
|
+
tableName
|
|
58
|
+
});
|
|
59
|
+
const entry = await activeProvider.get(key);
|
|
60
|
+
if (!entry) {
|
|
61
|
+
runtime.parsedValues.delete(key);
|
|
62
|
+
}
|
|
63
|
+
const fresh = entry ? Date.now() < entry.createdAt + entry.ttlMs : false;
|
|
64
|
+
const staleAcceptable = entry ? Date.now() < entry.createdAt + entry.ttlMs + entry.staleTtlMs : false;
|
|
65
|
+
const deserialize = mergedOptions.deserialize || runtime.deserialize;
|
|
66
|
+
const serialize = mergedOptions.serialize || runtime.serialize;
|
|
67
|
+
const respondFromCache = async (cacheEntry, status) => {
|
|
68
|
+
const memoized = runtime.parsedValues.get(key);
|
|
69
|
+
let rows;
|
|
70
|
+
if (memoized && memoized.createdAt === cacheEntry.createdAt) {
|
|
71
|
+
rows = memoized.rows;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
rows = await deserialize(cacheEntry.value);
|
|
75
|
+
runtime.parsedValues.set(key, { createdAt: cacheEntry.createdAt, rows, tags: cacheEntry.tags });
|
|
76
|
+
}
|
|
77
|
+
const cacheAge = Date.now() - cacheEntry.createdAt;
|
|
78
|
+
if (status === 'hit') {
|
|
79
|
+
runtime.stats.hits += 1;
|
|
80
|
+
}
|
|
81
|
+
else if (status === 'stale-hit') {
|
|
82
|
+
runtime.stats.staleHits += 1;
|
|
83
|
+
}
|
|
84
|
+
await logCacheHit({
|
|
85
|
+
sql,
|
|
86
|
+
parameters,
|
|
87
|
+
status,
|
|
88
|
+
cacheKey: key,
|
|
89
|
+
options: mergedOptions,
|
|
90
|
+
rowCount: cacheEntry.rowCount ?? rows.length,
|
|
91
|
+
ageMs: cacheAge,
|
|
92
|
+
queryId: options?.queryId
|
|
93
|
+
});
|
|
94
|
+
return rows;
|
|
95
|
+
};
|
|
96
|
+
if (mode === 'cache-first') {
|
|
97
|
+
if (entry && fresh) {
|
|
98
|
+
return respondFromCache(entry, 'hit');
|
|
99
|
+
}
|
|
100
|
+
runtime.stats.misses += 1;
|
|
101
|
+
return fetchAndStore('miss');
|
|
102
|
+
}
|
|
103
|
+
if (mode === 'stale-while-revalidate') {
|
|
104
|
+
if (entry && fresh) {
|
|
105
|
+
return respondFromCache(entry, 'hit');
|
|
106
|
+
}
|
|
107
|
+
if (entry && staleAcceptable) {
|
|
108
|
+
scheduleRevalidation();
|
|
109
|
+
return respondFromCache(entry, 'stale-hit');
|
|
110
|
+
}
|
|
111
|
+
runtime.stats.misses += 1;
|
|
112
|
+
return fetchAndStore('miss');
|
|
113
|
+
}
|
|
114
|
+
if (mode === 'network-first') {
|
|
115
|
+
try {
|
|
116
|
+
runtime.stats.misses += 1;
|
|
117
|
+
return await fetchAndStore('miss');
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (mergedOptions.staleIfError && entry && staleAcceptable) {
|
|
121
|
+
return respondFromCache(entry, 'stale-hit');
|
|
122
|
+
}
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return runWithoutCache('bypass');
|
|
127
|
+
async function fetchAndStore(cacheStatus) {
|
|
128
|
+
if (mergedOptions.dedupe !== false && runtime.inFlight.has(key)) {
|
|
129
|
+
return runtime.inFlight.get(key);
|
|
130
|
+
}
|
|
131
|
+
const promise = (async () => {
|
|
132
|
+
const rows = await builder.getExecutor().execute({
|
|
133
|
+
queryId: options?.queryId,
|
|
134
|
+
logContext: { cacheStatus, cacheKey: key, cacheMode: mode }
|
|
135
|
+
});
|
|
136
|
+
const encoded = await serialize(rows);
|
|
137
|
+
const ttlMs = mergedOptions.ttlMs ?? 0;
|
|
138
|
+
const staleTtlMs = mergedOptions.staleTtlMs ?? 0;
|
|
139
|
+
const cacheTimeMs = mergedOptions.cacheTimeMs ?? ttlMs + staleTtlMs;
|
|
140
|
+
const derivedTags = deriveTags(builder);
|
|
141
|
+
const tagSet = new Set([...(mergedOptions.tags || []), ...derivedTags]);
|
|
142
|
+
const newEntry = {
|
|
143
|
+
value: encoded.payload,
|
|
144
|
+
createdAt: Date.now(),
|
|
145
|
+
ttlMs,
|
|
146
|
+
staleTtlMs,
|
|
147
|
+
cacheTimeMs,
|
|
148
|
+
tags: Array.from(tagSet),
|
|
149
|
+
rowCount: rows.length,
|
|
150
|
+
byteSize: encoded.byteSize,
|
|
151
|
+
sqlFingerprint: key
|
|
152
|
+
};
|
|
153
|
+
await activeProvider.set(key, newEntry);
|
|
154
|
+
runtime.parsedValues.set(key, { createdAt: newEntry.createdAt, rows, tags: newEntry.tags });
|
|
155
|
+
return rows;
|
|
156
|
+
})();
|
|
157
|
+
if (mergedOptions.dedupe !== false) {
|
|
158
|
+
runtime.inFlight.set(key, promise);
|
|
159
|
+
promise.finally(() => runtime.inFlight.delete(key));
|
|
160
|
+
}
|
|
161
|
+
return promise;
|
|
162
|
+
}
|
|
163
|
+
function scheduleRevalidation() {
|
|
164
|
+
runtime.stats.revalidations += 1;
|
|
165
|
+
fetchAndStore('revalidate').catch(() => undefined);
|
|
166
|
+
}
|
|
167
|
+
function runWithoutCache(cacheStatus) {
|
|
168
|
+
if (provider) {
|
|
169
|
+
runtime.stats.misses += 1;
|
|
170
|
+
}
|
|
171
|
+
return builder.getExecutor().execute({
|
|
172
|
+
queryId: options?.queryId,
|
|
173
|
+
logContext: { cacheStatus, cacheMode: mode }
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { QueryRuntimeContext } from './runtime-context.js';
|
|
2
|
+
import type { CacheStats } from './types.js';
|
|
3
|
+
export declare class CacheController {
|
|
4
|
+
private context;
|
|
5
|
+
constructor(context: QueryRuntimeContext);
|
|
6
|
+
invalidateKey(key: string): Promise<void>;
|
|
7
|
+
invalidateTags(tags: string[]): Promise<void>;
|
|
8
|
+
clear(): Promise<void>;
|
|
9
|
+
warm(queries: Array<() => Promise<unknown>>): Promise<void>;
|
|
10
|
+
getStats(): CacheStats & {
|
|
11
|
+
hitRate: number;
|
|
12
|
+
};
|
|
13
|
+
private removeParsedValuesByTags;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=controller.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"controller.d.ts","sourceRoot":"","sources":["../../../src/core/cache/controller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAG7C,qBAAa,eAAe;IACd,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,mBAAmB;IAE1C,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAe7C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAOtB,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAKjE,QAAQ,IAAI,UAAU,GAAG;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE;IAO5C,OAAO,CAAC,wBAAwB;CAWjC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { logger } from '../utils/logger.js';
|
|
2
|
+
export class CacheController {
|
|
3
|
+
context;
|
|
4
|
+
constructor(context) {
|
|
5
|
+
this.context = context;
|
|
6
|
+
}
|
|
7
|
+
async invalidateKey(key) {
|
|
8
|
+
this.context.parsedValues.delete(key);
|
|
9
|
+
if (!this.context.provider)
|
|
10
|
+
return;
|
|
11
|
+
await this.context.provider.delete(key);
|
|
12
|
+
}
|
|
13
|
+
async invalidateTags(tags) {
|
|
14
|
+
if (!tags.length)
|
|
15
|
+
return;
|
|
16
|
+
const deleteByTag = this.context.provider?.deleteByTag;
|
|
17
|
+
if (!deleteByTag) {
|
|
18
|
+
logger.warn('Cache provider does not support tag invalidation. Tags ignored.', {
|
|
19
|
+
namespace: this.context.namespace,
|
|
20
|
+
tags
|
|
21
|
+
});
|
|
22
|
+
this.removeParsedValuesByTags(tags);
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
await Promise.all(tags.map(tag => deleteByTag.call(this.context.provider, this.context.namespace, tag)));
|
|
26
|
+
this.removeParsedValuesByTags(tags);
|
|
27
|
+
}
|
|
28
|
+
async clear() {
|
|
29
|
+
if (this.context.provider?.clearNamespace) {
|
|
30
|
+
await this.context.provider.clearNamespace(this.context.namespace);
|
|
31
|
+
}
|
|
32
|
+
this.context.parsedValues.clear();
|
|
33
|
+
}
|
|
34
|
+
async warm(queries) {
|
|
35
|
+
if (!queries.length)
|
|
36
|
+
return;
|
|
37
|
+
await Promise.all(queries.map(query => query()));
|
|
38
|
+
}
|
|
39
|
+
getStats() {
|
|
40
|
+
const stats = { ...this.context.stats };
|
|
41
|
+
const total = stats.hits + stats.misses + stats.staleHits;
|
|
42
|
+
const hitRate = total > 0 ? (stats.hits + stats.staleHits) / total : 0;
|
|
43
|
+
return { ...stats, hitRate };
|
|
44
|
+
}
|
|
45
|
+
removeParsedValuesByTags(tags) {
|
|
46
|
+
if (!tags.length)
|
|
47
|
+
return;
|
|
48
|
+
const target = new Set(tags);
|
|
49
|
+
for (const [key, record] of this.context.parsedValues) {
|
|
50
|
+
if (!record.tags?.length)
|
|
51
|
+
continue;
|
|
52
|
+
const intersects = record.tags.some(tag => target.has(tag));
|
|
53
|
+
if (intersects) {
|
|
54
|
+
this.context.parsedValues.delete(key);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
interface CacheKeyInput {
|
|
2
|
+
namespace: string;
|
|
3
|
+
sql: string;
|
|
4
|
+
parameters: unknown[];
|
|
5
|
+
settings?: Record<string, unknown> | undefined;
|
|
6
|
+
version?: string;
|
|
7
|
+
tableName?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare function computeCacheKey({ namespace, sql, parameters, settings, version, tableName }: CacheKeyInput): string;
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=key.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"key.d.ts","sourceRoot":"","sources":["../../../src/core/cache/key.ts"],"names":[],"mappings":"AAEA,UAAU,aAAa;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,OAAO,EAAE,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAqBD,wBAAgB,eAAe,CAAC,EAC9B,SAAS,EACT,GAAG,EACH,UAAU,EACV,QAAQ,EACR,OAAc,EACd,SAAS,EACV,EAAE,aAAa,GAAG,MAAM,CAOxB"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { stableStringify } from './serialization.js';
|
|
2
|
+
const FNV_OFFSET = BigInt('0xcbf29ce484222325');
|
|
3
|
+
const FNV_PRIME = BigInt('0x100000001b3');
|
|
4
|
+
const FNV_MOD = BigInt('0x10000000000000000');
|
|
5
|
+
function fnv1a64(value) {
|
|
6
|
+
let hash = FNV_OFFSET;
|
|
7
|
+
for (let i = 0; i < value.length; i++) {
|
|
8
|
+
hash ^= BigInt(value.charCodeAt(i));
|
|
9
|
+
hash = (hash * FNV_PRIME) % FNV_MOD;
|
|
10
|
+
}
|
|
11
|
+
return hash.toString(16);
|
|
12
|
+
}
|
|
13
|
+
function formatSegment(segment) {
|
|
14
|
+
if (!segment)
|
|
15
|
+
return 'query';
|
|
16
|
+
const sanitized = segment.replace(/[^a-zA-Z0-9_-]/g, '-');
|
|
17
|
+
return sanitized.slice(0, 48) || 'query';
|
|
18
|
+
}
|
|
19
|
+
export function computeCacheKey({ namespace, sql, parameters, settings, version = 'v1', tableName }) {
|
|
20
|
+
const serializedParams = stableStringify(parameters);
|
|
21
|
+
const serializedSettings = stableStringify(settings || null);
|
|
22
|
+
const material = `${sql}\n${serializedParams}\n${serializedSettings}`;
|
|
23
|
+
const digest = fnv1a64(material);
|
|
24
|
+
const tableSegment = formatSegment(tableName);
|
|
25
|
+
return `hq:${version}:${namespace}:${tableSegment}:${digest}`;
|
|
26
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { CacheEntry, CacheProvider } from '../types.js';
|
|
2
|
+
export interface MemoryLRUCacheOptions {
|
|
3
|
+
maxEntries?: number;
|
|
4
|
+
maxBytes?: number;
|
|
5
|
+
cleanupIntervalMs?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare class MemoryCacheProvider implements CacheProvider {
|
|
8
|
+
private entries;
|
|
9
|
+
private tagIndex;
|
|
10
|
+
private currentBytes;
|
|
11
|
+
private maxEntries;
|
|
12
|
+
private maxBytes;
|
|
13
|
+
private cleanupIntervalMs;
|
|
14
|
+
private cleanupTimer?;
|
|
15
|
+
constructor(options?: MemoryLRUCacheOptions);
|
|
16
|
+
dispose(): void;
|
|
17
|
+
get(key: string): Promise<CacheEntry | null>;
|
|
18
|
+
set(key: string, entry: CacheEntry): Promise<void>;
|
|
19
|
+
delete(key: string): Promise<void>;
|
|
20
|
+
deleteByTag(namespace: string, tag: string): Promise<void>;
|
|
21
|
+
clearNamespace(namespace: string): Promise<void>;
|
|
22
|
+
private touch;
|
|
23
|
+
private enforceLimits;
|
|
24
|
+
private cleanup;
|
|
25
|
+
private indexTags;
|
|
26
|
+
private unindexTags;
|
|
27
|
+
private getTagIndexKey;
|
|
28
|
+
private cleanupTagIndex;
|
|
29
|
+
}
|
|
30
|
+
export { MemoryCacheProvider as MemoryLRUCacheProvider };
|
|
31
|
+
//# sourceMappingURL=memory-lru.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-lru.d.ts","sourceRoot":"","sources":["../../../../src/core/cache/providers/memory-lru.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE7D,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAgBD,qBAAa,mBAAoB,YAAW,aAAa;IACvD,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,YAAY,CAAC,CAAiC;gBAE1C,OAAO,GAAE,qBAA0B;IAa/C,OAAO,IAAI,IAAI;IAOT,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAY5C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAYlD,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQlC,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAU1D,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAUtD,OAAO,CAAC,KAAK;IAKb,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,OAAO;IAWf,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,eAAe;CAYxB;AAED,OAAO,EAAE,mBAAmB,IAAI,sBAAsB,EAAE,CAAC"}
|