jinaga-server 3.5.1 → 3.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authorization/authorization-keystore.js +2 -2
- package/dist/authorization/authorization-keystore.js.map +1 -1
- package/dist/http/csv-metadata.d.ts +24 -0
- package/dist/http/csv-metadata.js +3 -0
- package/dist/http/csv-metadata.js.map +1 -0
- package/dist/http/csv-validator.d.ts +23 -0
- package/dist/http/csv-validator.js +83 -0
- package/dist/http/csv-validator.js.map +1 -0
- package/dist/http/output-formatters.d.ts +28 -0
- package/dist/http/output-formatters.js +197 -0
- package/dist/http/output-formatters.js.map +1 -0
- package/dist/http/result-stream.d.ts +35 -0
- package/dist/http/result-stream.js +78 -0
- package/dist/http/result-stream.js.map +1 -0
- package/dist/http/router.d.ts +7 -1
- package/dist/http/router.js +276 -70
- package/dist/http/router.js.map +1 -1
- package/dist/http/stream.d.ts +3 -0
- package/dist/http/stream.js +79 -5
- package/dist/http/stream.js.map +1 -1
- package/dist/postgres/postgres-store.js +7 -5
- package/dist/postgres/postgres-store.js.map +1 -1
- package/dist/postgres/purge-sql.js +15 -2
- package/dist/postgres/purge-sql.js.map +1 -1
- package/dist/postgres/specification-result-sql.d.ts +2 -0
- package/dist/postgres/specification-result-sql.js +49 -5
- package/dist/postgres/specification-result-sql.js.map +1 -1
- package/dist/postgres/specification-sql.js +28 -6
- package/dist/postgres/specification-sql.js.map +1 -1
- package/docs/analysis/DIAGNOSTIC_LOGGING_README.md +201 -0
- package/docs/analysis/security-audit-feeds-hash-handler.md +407 -0
- package/docs/design/feeds-persistent-proxy-plan.md +294 -0
- package/integration-test/integration-test/jinaga-test/package-lock.json +6 -0
- package/integration-test/jinaga-test/Dockerfile +1 -6
- package/integration-test/jinaga-test/http-test-helpers.js +77 -0
- package/integration-test/jinaga-test/jinaga.test.js +9 -9
- package/integration-test/jinaga-test/package-lock.json +329 -32
- package/integration-test/jinaga-test/package.json +5 -1
- package/integration-test/jinaga-test/read-csv.test.js +330 -0
- package/integration-test/jinaga-test/read-endpoint.test.js +442 -0
- package/integration-test/jinaga-test/read-performance.test.js +221 -0
- package/integration-test/jinaga-test/read-streaming.test.js +199 -0
- package/integration-test/jinaga-test/specification-conditions.test.js +147 -0
- package/integration-test/jinaga-test/test-helpers.js +12 -0
- package/integration-test/jinaga-test/time-projection.test.js +235 -0
- package/package.json +3 -2
- package/src/authorization/authorization-keystore.ts +4 -4
- package/src/http/csv-metadata.ts +34 -0
- package/src/http/csv-validator.ts +90 -0
- package/src/http/output-formatters.ts +196 -0
- package/src/http/result-stream.ts +69 -0
- package/src/http/router.ts +344 -62
- package/src/http/stream.ts +97 -5
- package/src/postgres/postgres-store.ts +13 -6
- package/src/postgres/purge-sql.ts +18 -2
- package/src/postgres/specification-result-sql.ts +70 -8
- package/src/postgres/specification-sql.ts +36 -8
- package/test/http/csv-validatorSpec.ts +205 -0
- package/test/http/result-streamSpec.ts +151 -0
- package/test/postgres/postgresReadSpec.ts +192 -14
- package/test/postgres/postgresSpecificationSpec.ts +3 -3
- package/test/postgres/specificationConditionsSpec.ts +268 -0
- package/test-concurrent-streams.js +219 -0
- package/documentation/streaming-feeds-full-results-solution.md +0 -230
package/src/http/stream.ts
CHANGED
|
@@ -3,30 +3,122 @@ export class Stream<T> {
|
|
|
3
3
|
private handlers: ((data: T) => void)[] = [];
|
|
4
4
|
private doneHandlers: (() => void)[] = [];
|
|
5
5
|
private closed = false;
|
|
6
|
+
private readonly streamId: string;
|
|
7
|
+
private feedCount = 0;
|
|
8
|
+
|
|
9
|
+
constructor() {
|
|
10
|
+
this.streamId = Math.random().toString(36).substring(2, 10);
|
|
11
|
+
console.log(`[Stream:${this.streamId}] Created new stream`);
|
|
12
|
+
}
|
|
6
13
|
|
|
7
14
|
next(handler: (data: T) => void): Stream<T> {
|
|
15
|
+
if (this.closed) {
|
|
16
|
+
console.warn(`[Stream:${this.streamId}] Attempted to add handler to closed stream`);
|
|
17
|
+
return this;
|
|
18
|
+
}
|
|
19
|
+
|
|
8
20
|
this.handlers.push(handler);
|
|
9
|
-
this.queue.
|
|
21
|
+
const queueLength = this.queue.length;
|
|
22
|
+
console.log(`[Stream:${this.streamId}] Added handler - Total handlers: ${this.handlers.length}, Queued items: ${queueLength}`);
|
|
23
|
+
|
|
24
|
+
// Replay queued data to new handler
|
|
25
|
+
this.queue.forEach((data, index) => {
|
|
26
|
+
try {
|
|
27
|
+
handler(data);
|
|
28
|
+
console.log(`[Stream:${this.streamId}] Replayed queued item ${index + 1}/${queueLength} to new handler`);
|
|
29
|
+
} catch (error) {
|
|
30
|
+
console.error(`[Stream:${this.streamId}] Error replaying queued item ${index + 1}: ${error}`);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
10
34
|
return this;
|
|
11
35
|
}
|
|
12
36
|
|
|
13
37
|
done(handler: () => void): Stream<T> {
|
|
38
|
+
if (this.closed) {
|
|
39
|
+
console.warn(`[Stream:${this.streamId}] Attempted to add done handler to closed stream - executing immediately`);
|
|
40
|
+
try {
|
|
41
|
+
handler();
|
|
42
|
+
} catch (error) {
|
|
43
|
+
console.error(`[Stream:${this.streamId}] Error in immediate done handler: ${error}`);
|
|
44
|
+
}
|
|
45
|
+
return this;
|
|
46
|
+
}
|
|
47
|
+
|
|
14
48
|
this.doneHandlers.push(handler);
|
|
49
|
+
console.log(`[Stream:${this.streamId}] Added done handler - Total done handlers: ${this.doneHandlers.length}`);
|
|
15
50
|
return this;
|
|
16
51
|
}
|
|
17
52
|
|
|
18
53
|
close(): void {
|
|
54
|
+
if (this.closed) {
|
|
55
|
+
console.warn(`[Stream:${this.streamId}] Attempted to close already closed stream`);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
console.log(`[Stream:${this.streamId}] CLOSING stream - Handlers: ${this.handlers.length}, Done handlers: ${this.doneHandlers.length}, Queue: ${this.queue.length}, Feed count: ${this.feedCount}`);
|
|
60
|
+
|
|
19
61
|
this.closed = true;
|
|
20
|
-
|
|
62
|
+
|
|
63
|
+
// Execute done handlers
|
|
64
|
+
const doneHandlerCount = this.doneHandlers.length;
|
|
65
|
+
this.doneHandlers.forEach((handler, index) => {
|
|
66
|
+
try {
|
|
67
|
+
handler();
|
|
68
|
+
console.log(`[Stream:${this.streamId}] Executed done handler ${index + 1}/${doneHandlerCount}`);
|
|
69
|
+
} catch (error) {
|
|
70
|
+
console.error(`[Stream:${this.streamId}] Error in done handler ${index + 1}: ${error}`);
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// Clear all references
|
|
21
75
|
this.queue = [];
|
|
22
76
|
this.handlers = [];
|
|
23
77
|
this.doneHandlers = [];
|
|
78
|
+
|
|
79
|
+
console.log(`[Stream:${this.streamId}] Stream closed and cleaned up`);
|
|
24
80
|
}
|
|
25
81
|
|
|
26
82
|
feed(data: any): void {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
83
|
+
const feedStart = Date.now();
|
|
84
|
+
|
|
85
|
+
if (this.closed) {
|
|
86
|
+
console.warn(`[Stream:${this.streamId}] Attempted to feed data to closed stream - Data type: ${typeof data}`);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
this.feedCount++;
|
|
91
|
+
const handlerCount = this.handlers.length;
|
|
92
|
+
|
|
93
|
+
console.log(`[Stream:${this.streamId}] FEEDING data - Feed #${this.feedCount}, Handlers: ${handlerCount}, Queue size before: ${this.queue.length}`);
|
|
94
|
+
|
|
95
|
+
this.queue.push(data);
|
|
96
|
+
|
|
97
|
+
// Notify all handlers
|
|
98
|
+
let successCount = 0;
|
|
99
|
+
let errorCount = 0;
|
|
100
|
+
|
|
101
|
+
this.handlers.forEach((handler, index) => {
|
|
102
|
+
try {
|
|
103
|
+
const handlerStart = Date.now();
|
|
104
|
+
handler(data);
|
|
105
|
+
const handlerDuration = Date.now() - handlerStart;
|
|
106
|
+
successCount++;
|
|
107
|
+
|
|
108
|
+
if (handlerDuration > 50) {
|
|
109
|
+
console.warn(`[Stream:${this.streamId}] SLOW handler - Handler ${index + 1}/${handlerCount}, Duration: ${handlerDuration}ms`);
|
|
110
|
+
}
|
|
111
|
+
} catch (error) {
|
|
112
|
+
errorCount++;
|
|
113
|
+
console.error(`[Stream:${this.streamId}] ERROR in handler ${index + 1}/${handlerCount}: ${error}`);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
const totalDuration = Date.now() - feedStart;
|
|
118
|
+
console.log(`[Stream:${this.streamId}] Feed complete - Success: ${successCount}, Errors: ${errorCount}, Queue size after: ${this.queue.length}, Duration: ${totalDuration}ms`);
|
|
119
|
+
|
|
120
|
+
if (totalDuration > 100) {
|
|
121
|
+
console.warn(`[Stream:${this.streamId}] SLOW feed operation - Duration: ${totalDuration}ms`);
|
|
30
122
|
}
|
|
31
123
|
}
|
|
32
124
|
}
|
|
@@ -46,6 +46,10 @@ import { ResultSetFact, ResultSetRow, ResultSetTree, resultSqlFromSpecification,
|
|
|
46
46
|
import { sqlFromFeed } from "./specification-sql";
|
|
47
47
|
import { purgeDescendantsSql, purgeSqlFromSpecification } from "./purge-sql";
|
|
48
48
|
|
|
49
|
+
interface PostgresFactRecord extends FactRecord {
|
|
50
|
+
timestamp: Date; // Maps to date_learned column
|
|
51
|
+
}
|
|
52
|
+
|
|
49
53
|
interface FactTypeResult {
|
|
50
54
|
rows: {
|
|
51
55
|
fact_type_id: number;
|
|
@@ -91,6 +95,7 @@ interface AncestorResult {
|
|
|
91
95
|
data: string;
|
|
92
96
|
public_key: string;
|
|
93
97
|
signature: string;
|
|
98
|
+
date_learned: Date;
|
|
94
99
|
}[];
|
|
95
100
|
}
|
|
96
101
|
|
|
@@ -353,15 +358,15 @@ export class PostgresStore implements Storage {
|
|
|
353
358
|
const factParameters = flatten(references, (f) =>
|
|
354
359
|
[f.hash, factTypes.get(f.type)]);
|
|
355
360
|
const sql =
|
|
356
|
-
'SELECT f.fact_type_id, f.name, f.hash, f.data, s.public_key, s.signature ' +
|
|
357
|
-
'FROM (SELECT f.fact_id, f.fact_type_id, t.name, f.hash, f.data ' +
|
|
361
|
+
'SELECT f.fact_type_id, f.name, f.hash, f.data, f.date_learned, s.public_key, s.signature ' +
|
|
362
|
+
'FROM (SELECT f.fact_id, f.fact_type_id, t.name, f.hash, f.data, f.date_learned ' +
|
|
358
363
|
`FROM ${this.schema}.fact f ` +
|
|
359
364
|
`JOIN ${this.schema}.fact_type t ` +
|
|
360
365
|
' ON f.fact_type_id = t.fact_type_id ' +
|
|
361
366
|
'JOIN (VALUES ' + factValues.join(', ') + ') AS v (hash, fact_type_id) ' +
|
|
362
367
|
' ON v.fact_type_id = f.fact_type_id AND v.hash = f.hash ' +
|
|
363
368
|
'UNION ' +
|
|
364
|
-
'SELECT f2.fact_id, f2.fact_type_id, t.name, f2.hash, f2.data ' +
|
|
369
|
+
'SELECT f2.fact_id, f2.fact_type_id, t.name, f2.hash, f2.data, f2.date_learned ' +
|
|
365
370
|
`FROM ${this.schema}.fact f1 ` +
|
|
366
371
|
'JOIN (VALUES ' + factValues.join(', ') + ') AS v (hash, fact_type_id) ' +
|
|
367
372
|
' ON v.fact_type_id = f1.fact_type_id AND v.hash = f1.hash ' +
|
|
@@ -387,11 +392,12 @@ export class PostgresStore implements Storage {
|
|
|
387
392
|
this.factTypeMap = mergeFactTypes(this.factTypeMap, resultFactTypes);
|
|
388
393
|
const records = result.rows.map((r) => {
|
|
389
394
|
const { fields, predecessors }: { fields: {}, predecessors: PredecessorCollection } = r.data as any;
|
|
390
|
-
const fact:
|
|
395
|
+
const fact: PostgresFactRecord = {
|
|
391
396
|
type: r.name,
|
|
392
397
|
hash: r.hash,
|
|
393
398
|
fields,
|
|
394
|
-
predecessors
|
|
399
|
+
predecessors,
|
|
400
|
+
timestamp: r.date_learned
|
|
395
401
|
};
|
|
396
402
|
const signature: FactSignature = {
|
|
397
403
|
publicKey: r.public_key,
|
|
@@ -523,7 +529,8 @@ async function executeQueryTree(sqlQueryTree: SqlQueryTree, connection: PoolClie
|
|
|
523
529
|
const fact: ResultSetFact = {
|
|
524
530
|
hash: dataRow[`hash${label.index}`],
|
|
525
531
|
factId: dataRow[`id${label.index}`],
|
|
526
|
-
data: dataRow[`data${label.index}`]
|
|
532
|
+
data: dataRow[`data${label.index}`],
|
|
533
|
+
timestamp: dataRow[`timestamp${label.index}`],
|
|
527
534
|
};
|
|
528
535
|
return {
|
|
529
536
|
...acc,
|
|
@@ -47,9 +47,25 @@ export function purgeSqlFromSpecification(specification: Specification, factType
|
|
|
47
47
|
|
|
48
48
|
let queryDescription = QueryDescription.unsatisfiable;
|
|
49
49
|
let knownFacts: FactByLabel = {};
|
|
50
|
-
const given = specification.given;
|
|
51
|
-
const start: FactReference[] = specification.given.map(g => ({ type: g.type, hash: 'xxxxx' }));
|
|
50
|
+
const given = specification.given.map(g => g.label);
|
|
51
|
+
const start: FactReference[] = specification.given.map(g => ({ type: g.label.type, hash: 'xxxxx' }));
|
|
52
|
+
|
|
53
|
+
// Process main matches
|
|
52
54
|
({ queryDescription, knownFacts } = queryDescriptionBuilder.addEdges(queryDescription, given, start, knownFacts, [], specification.matches));
|
|
55
|
+
|
|
56
|
+
// Process conditions from given
|
|
57
|
+
for (const givenItem of specification.given) {
|
|
58
|
+
if (givenItem.conditions && givenItem.conditions.length > 0) {
|
|
59
|
+
for (const condition of givenItem.conditions) {
|
|
60
|
+
const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
|
|
61
|
+
const { queryDescription: queryDescriptionConditional } = queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, given, start, knownFacts, conditionalPath, condition.matches);
|
|
62
|
+
|
|
63
|
+
if (queryDescriptionConditional.isSatisfiable()) {
|
|
64
|
+
queryDescription = queryDescriptionConditional;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
53
69
|
|
|
54
70
|
if (!queryDescription.isSatisfiable()) {
|
|
55
71
|
return null;
|
|
@@ -6,7 +6,6 @@ import {
|
|
|
6
6
|
FieldProjection,
|
|
7
7
|
HashProjection,
|
|
8
8
|
hydrateFromTree,
|
|
9
|
-
Label,
|
|
10
9
|
Match,
|
|
11
10
|
PredecessorCollection,
|
|
12
11
|
ProjectedResult,
|
|
@@ -14,7 +13,9 @@ import {
|
|
|
14
13
|
ReferencesByName,
|
|
15
14
|
SingularProjection,
|
|
16
15
|
Specification,
|
|
16
|
+
SpecificationGiven,
|
|
17
17
|
SpecificationProjection,
|
|
18
|
+
TimeProjection,
|
|
18
19
|
validateGiven
|
|
19
20
|
} from "jinaga";
|
|
20
21
|
|
|
@@ -24,7 +25,7 @@ import { EdgeDescription, ExistentialConditionDescription, FactByLabel, QueryDes
|
|
|
24
25
|
function generateResultSqlQuery(queryDescription: QueryDescription, schema: string): SpecificationSqlQuery {
|
|
25
26
|
const allLabels = [ ...queryDescription.inputs, ...queryDescription.outputs ];
|
|
26
27
|
const columns = allLabels
|
|
27
|
-
.map(label => `f${label.factIndex}.hash as hash${label.factIndex}, f${label.factIndex}.fact_id as id${label.factIndex}, f${label.factIndex}.data as data${label.factIndex}`)
|
|
28
|
+
.map(label => `f${label.factIndex}.hash as hash${label.factIndex}, f${label.factIndex}.fact_id as id${label.factIndex}, f${label.factIndex}.data as data${label.factIndex}, f${label.factIndex}.date_learned as timestamp${label.factIndex}`)
|
|
28
29
|
.join(", ");
|
|
29
30
|
const firstEdge = queryDescription.edges[0];
|
|
30
31
|
const predecessorFact = queryDescription.inputs.find(i => i.factIndex === firstEdge.predecessorFactIndex);
|
|
@@ -32,6 +33,8 @@ function generateResultSqlQuery(queryDescription: QueryDescription, schema: stri
|
|
|
32
33
|
const firstFactIndex = predecessorFact ? predecessorFact.factIndex : successorFact!.factIndex;
|
|
33
34
|
const writtenFactIndexes = new Set<number>().add(firstFactIndex);
|
|
34
35
|
const joins: string[] = generateJoins(queryDescription.edges, writtenFactIndexes, schema);
|
|
36
|
+
// Add all input fact indexes to writtenFactIndexes since they're available in the FROM clause
|
|
37
|
+
queryDescription.inputs.forEach(input => writtenFactIndexes.add(input.factIndex));
|
|
35
38
|
const inputWhereClauses = queryDescription.inputs
|
|
36
39
|
.map(input => `f${input.factIndex}.fact_type_id = $${input.factTypeParameter} AND f${input.factIndex}.hash = $${input.factHashParameter}`)
|
|
37
40
|
.join(" AND ");
|
|
@@ -197,6 +200,7 @@ export interface ResultSetFact {
|
|
|
197
200
|
hash: string;
|
|
198
201
|
factId: number;
|
|
199
202
|
data: ResultSetData;
|
|
203
|
+
timestamp: Date;
|
|
200
204
|
}
|
|
201
205
|
|
|
202
206
|
export interface ResultSetRow {
|
|
@@ -411,6 +415,9 @@ export class ResultComposer {
|
|
|
411
415
|
else if (this.resultProjection.type === "fact") {
|
|
412
416
|
return this.factValue(this.resultProjection, row, factRecords);
|
|
413
417
|
}
|
|
418
|
+
else if (this.resultProjection.type === "time") {
|
|
419
|
+
return this.timeValue(this.resultProjection, row);
|
|
420
|
+
}
|
|
414
421
|
else {
|
|
415
422
|
const _exhaustiveCheck: never = this.resultProjection;
|
|
416
423
|
throw new Error(`Unknown projection type ${(this.resultProjection as any).type}`);
|
|
@@ -439,6 +446,9 @@ export class ResultComposer {
|
|
|
439
446
|
// This should have already been taken care of
|
|
440
447
|
return null;
|
|
441
448
|
}
|
|
449
|
+
else if (projection.type === "time") {
|
|
450
|
+
return this.timeValue(projection, row);
|
|
451
|
+
}
|
|
442
452
|
else {
|
|
443
453
|
const _exhaustiveCheck: never = projection;
|
|
444
454
|
throw new Error(`Unknown projection type ${(projection as any).type}`);
|
|
@@ -465,6 +475,11 @@ export class ResultComposer {
|
|
|
465
475
|
return fact;
|
|
466
476
|
}
|
|
467
477
|
|
|
478
|
+
private timeValue(projection: TimeProjection, row: ResultSetRow): any {
|
|
479
|
+
const label = this.getLabel(projection.label);
|
|
480
|
+
return row[label.index].timestamp;
|
|
481
|
+
}
|
|
482
|
+
|
|
468
483
|
private getLabel(name: string) {
|
|
469
484
|
const label = this.sqlQuery.labels.find(label => label.name === name);
|
|
470
485
|
if (!label) {
|
|
@@ -487,15 +502,62 @@ class ResultDescriptionBuilder {
|
|
|
487
502
|
buildDescription(start: FactReference[], specification: Specification): ResultDescription {
|
|
488
503
|
validateGiven(start, specification);
|
|
489
504
|
|
|
490
|
-
|
|
505
|
+
const labels = specification.given.map(g => g.label);
|
|
506
|
+
let queryDescription = QueryDescription.unsatisfiable;
|
|
507
|
+
let knownFacts: FactByLabel = {};
|
|
508
|
+
|
|
509
|
+
// First process main matches to build the result description
|
|
510
|
+
const resultDescription = this.createResultDescription(queryDescription, specification.given, start, specification.matches, specification.projection, knownFacts, []);
|
|
511
|
+
|
|
512
|
+
// Then process conditions from given if we have a satisfiable query
|
|
513
|
+
if (resultDescription.queryDescription.isSatisfiable()) {
|
|
514
|
+
queryDescription = resultDescription.queryDescription;
|
|
515
|
+
knownFacts = resultDescription.queryDescription.outputs.reduce((acc, output) => ({
|
|
516
|
+
...acc,
|
|
517
|
+
[output.label]: { factIndex: output.factIndex, type: output.type }
|
|
518
|
+
}), {} as FactByLabel);
|
|
519
|
+
|
|
520
|
+
// Also include inputs in knownFacts
|
|
521
|
+
for (const input of resultDescription.queryDescription.inputs) {
|
|
522
|
+
if (!knownFacts[input.label]) {
|
|
523
|
+
knownFacts[input.label] = { factIndex: input.factIndex, type: input.type };
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// Process conditions from each given
|
|
528
|
+
for (const givenItem of specification.given) {
|
|
529
|
+
if (givenItem.conditions && givenItem.conditions.length > 0) {
|
|
530
|
+
for (const condition of givenItem.conditions) {
|
|
531
|
+
const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
|
|
532
|
+
const { queryDescription: queryDescriptionConditional } = this.queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, labels, start, knownFacts, conditionalPath, condition.matches);
|
|
533
|
+
|
|
534
|
+
if (queryDescriptionConditional.isSatisfiable()) {
|
|
535
|
+
queryDescription = queryDescriptionConditional;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// Return updated result description with conditions applied
|
|
542
|
+
return {
|
|
543
|
+
...resultDescription,
|
|
544
|
+
queryDescription
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
return resultDescription;
|
|
491
549
|
}
|
|
492
550
|
|
|
493
|
-
private createResultDescription(queryDescription: QueryDescription, given:
|
|
494
|
-
const givenTuple = given.reduce((acc,
|
|
551
|
+
private createResultDescription(queryDescription: QueryDescription, given: SpecificationGiven[], start: FactReference[], matches: Match[], projection: Projection, knownFacts: FactByLabel, path: number[]): ResultDescription {
|
|
552
|
+
const givenTuple = given.reduce((acc, givenItem, index) => ({
|
|
495
553
|
...acc,
|
|
496
|
-
[label.name]: start[index]
|
|
554
|
+
[givenItem.label.name]: start[index]
|
|
497
555
|
}), {} as ReferencesByName);
|
|
498
|
-
|
|
556
|
+
|
|
557
|
+
// Process main matches
|
|
558
|
+
const labels = given.map(g => g.label);
|
|
559
|
+
({ queryDescription, knownFacts } = this.queryDescriptionBuilder.addEdges(queryDescription, labels, start, knownFacts, path, matches));
|
|
560
|
+
|
|
499
561
|
if (!queryDescription.isSatisfiable()) {
|
|
500
562
|
// Abort the branch if the query is not satisfiable
|
|
501
563
|
return {
|
|
@@ -513,7 +575,7 @@ class ResultDescriptionBuilder {
|
|
|
513
575
|
const specificationProjections = projection.components
|
|
514
576
|
.filter(projection => projection.type === "specification") as ({ name: string } & SpecificationProjection)[];
|
|
515
577
|
const singularProjections = projection.components
|
|
516
|
-
.filter(projection => projection.type === "field" || projection.type === "hash" || projection.type === "fact") as ({ name: string } & SingularProjection)[];
|
|
578
|
+
.filter(projection => projection.type === "field" || projection.type === "hash" || projection.type === "fact" || projection.type === "time") as ({ name: string } & SingularProjection)[];
|
|
517
579
|
for (const child of specificationProjections) {
|
|
518
580
|
const childResultDescription = this.createResultDescription(queryDescription, given, start, child.matches, child.projection, knownFacts, []);
|
|
519
581
|
childResultDescriptions.push({
|
|
@@ -40,9 +40,13 @@ function generateSqlQuery(queryDescription: QueryDescription, schema: string, bo
|
|
|
40
40
|
.filter(c => c.exists === false))
|
|
41
41
|
.map(notExistsWhereClause => ` AND NOT EXISTS (${generateNotExistsWhereClause(schema, notExistsWhereClause, writtenFactIndexes)})`)
|
|
42
42
|
.join("");
|
|
43
|
+
const existsWhereClauses = (queryDescription.existentialConditions
|
|
44
|
+
.filter(c => c.exists === true))
|
|
45
|
+
.map(existsWhereClause => ` AND EXISTS (${generateNotExistsWhereClause(schema, existsWhereClause, writtenFactIndexes)})`)
|
|
46
|
+
.join("");
|
|
43
47
|
const bookmarkParameter = queryDescription.parameters.length + 1;
|
|
44
48
|
const limitParameter = bookmarkParameter + 1;
|
|
45
|
-
const sql = `SELECT ${hashes}, sort(array[${factIds}], 'desc') as bookmark FROM ${schema}.fact f${firstFactIndex}${joins.join("")} WHERE ${inputWhereClauses}${notExistsWhereClauses} AND sort(array[${factIds}], 'desc') > $${bookmarkParameter} ORDER BY bookmark ASC LIMIT $${limitParameter}`;
|
|
49
|
+
const sql = `SELECT ${hashes}, sort(array[${factIds}], 'desc') as bookmark FROM ${schema}.fact f${firstFactIndex}${joins.join("")} WHERE ${inputWhereClauses}${notExistsWhereClauses}${existsWhereClauses} AND sort(array[${factIds}], 'desc') > $${bookmarkParameter} ORDER BY bookmark ASC LIMIT $${limitParameter}`;
|
|
46
50
|
const bookmarkValue: number[] = parseBookmark(bookmark);
|
|
47
51
|
return {
|
|
48
52
|
sql,
|
|
@@ -165,7 +169,7 @@ export function sqlFromSpecification(start: FactReference[], schema: string, boo
|
|
|
165
169
|
validateGiven(start, specification);
|
|
166
170
|
|
|
167
171
|
const labeledStart = start.map((s, index) => ({
|
|
168
|
-
name: specification.given[index].name,
|
|
172
|
+
name: specification.given[index].label.name,
|
|
169
173
|
reference: s
|
|
170
174
|
})).reduce((map, s) => {
|
|
171
175
|
map.set(s.name, s.reference);
|
|
@@ -179,9 +183,9 @@ export function sqlFromSpecification(start: FactReference[], schema: string, boo
|
|
|
179
183
|
bookmark: bookmarks[index]
|
|
180
184
|
}));
|
|
181
185
|
const queryDescriptionsAndBookmarks = feedAndBookmark.map(fb => {
|
|
182
|
-
const feedStart = fb.feed.given.map(s => labeledStart.get(s.name)!);
|
|
186
|
+
const feedStart = fb.feed.given.map(s => labeledStart.get(s.label.name)!);
|
|
183
187
|
return {
|
|
184
|
-
queryDescription: buildQueryDescription(queryDescriptionBuilder, fb.feed, feedStart),
|
|
188
|
+
queryDescription: buildQueryDescription(queryDescriptionBuilder, fb.feed, feedStart, specification.given),
|
|
185
189
|
bookmark: fb.bookmark
|
|
186
190
|
};
|
|
187
191
|
});
|
|
@@ -202,11 +206,35 @@ export function sqlFromFeed(feed: Specification, start: FactReference[], schema:
|
|
|
202
206
|
return sql;
|
|
203
207
|
}
|
|
204
208
|
|
|
205
|
-
function buildQueryDescription(queryDescriptionBuilder: QueryDescriptionBuilder, specification: Specification, start: FactReference[]): QueryDescription {
|
|
206
|
-
const
|
|
209
|
+
function buildQueryDescription(queryDescriptionBuilder: QueryDescriptionBuilder, specification: Specification, start: FactReference[], originalGiven?: any[]): QueryDescription {
|
|
210
|
+
const labels = specification.given.map(g => g.label);
|
|
211
|
+
let knownFacts = {};
|
|
212
|
+
|
|
213
|
+
// Process main matches
|
|
214
|
+
let { queryDescription, knownFacts: updatedKnownFacts } = queryDescriptionBuilder.addEdges(
|
|
207
215
|
QueryDescription.unsatisfiable,
|
|
208
|
-
|
|
209
|
-
start,
|
|
216
|
+
labels,
|
|
217
|
+
start,
|
|
218
|
+
knownFacts,
|
|
219
|
+
[],
|
|
210
220
|
specification.matches);
|
|
221
|
+
knownFacts = updatedKnownFacts;
|
|
222
|
+
|
|
223
|
+
// Process conditions from given (use originalGiven if provided, otherwise use specification.given)
|
|
224
|
+
const givenToProcess = originalGiven || specification.given;
|
|
225
|
+
for (const givenItem of givenToProcess) {
|
|
226
|
+
if (givenItem.conditions && givenItem.conditions.length > 0) {
|
|
227
|
+
for (const condition of givenItem.conditions) {
|
|
228
|
+
const { query: queryDescriptionWithExistential, path: conditionalPath } = queryDescription.withExistentialCondition(condition.exists, []);
|
|
229
|
+
const { queryDescription: queryDescriptionConditional, knownFacts: updatedKnownFacts } = queryDescriptionBuilder.addEdges(queryDescriptionWithExistential, labels, start, knownFacts, conditionalPath, condition.matches);
|
|
230
|
+
|
|
231
|
+
if (queryDescriptionConditional.isSatisfiable()) {
|
|
232
|
+
queryDescription = queryDescriptionConditional;
|
|
233
|
+
knownFacts = updatedKnownFacts;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
211
239
|
return queryDescription;
|
|
212
240
|
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { validateSpecificationForCsv, extractValueByLabel, formatValueForCsv } from '../../src/http/csv-validator';
|
|
2
|
+
import { Specification, SpecificationParser, CompositeProjection } from 'jinaga';
|
|
3
|
+
|
|
4
|
+
describe('CSV Validator', () => {
|
|
5
|
+
describe('validateSpecificationForCsv', () => {
|
|
6
|
+
// Helper function to create a specification from declarative syntax
|
|
7
|
+
function GivenSpecificationFromInput(input: string): Specification {
|
|
8
|
+
const parser = new SpecificationParser(input);
|
|
9
|
+
return parser.parseSpecification();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
it('should accept flat field projections', () => {
|
|
13
|
+
const input = `(root: Root) {
|
|
14
|
+
successor: Successor [
|
|
15
|
+
successor->root: Root = root
|
|
16
|
+
]
|
|
17
|
+
} => {
|
|
18
|
+
name = successor.name
|
|
19
|
+
count = successor.count
|
|
20
|
+
}`;
|
|
21
|
+
const specification = GivenSpecificationFromInput(input);
|
|
22
|
+
|
|
23
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
24
|
+
|
|
25
|
+
expect(metadata.isValid).toBe(true);
|
|
26
|
+
expect(metadata.headers).toEqual(['name', 'count']);
|
|
27
|
+
expect(metadata.errors).toEqual([]);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
// Acceptance tests for future hash/timestamp support
|
|
31
|
+
it('should accept hash projections', () => {
|
|
32
|
+
const input = `(root: Root) {
|
|
33
|
+
successor: Successor [
|
|
34
|
+
successor->root: Root = root
|
|
35
|
+
]
|
|
36
|
+
} => {
|
|
37
|
+
hash = #successor
|
|
38
|
+
}`;
|
|
39
|
+
const specification = GivenSpecificationFromInput(input);
|
|
40
|
+
|
|
41
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
42
|
+
|
|
43
|
+
expect(metadata.errors).toEqual([]);
|
|
44
|
+
expect(metadata.isValid).toBe(true);
|
|
45
|
+
expect(metadata.headers).toEqual(['hash']);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('should accept timestamp projections', () => {
|
|
49
|
+
const input = `(root: Root) {
|
|
50
|
+
successor: Successor [
|
|
51
|
+
successor->root: Root = root
|
|
52
|
+
]
|
|
53
|
+
} => {
|
|
54
|
+
timestamp = @successor
|
|
55
|
+
}`;
|
|
56
|
+
const specification = GivenSpecificationFromInput(input);
|
|
57
|
+
|
|
58
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
59
|
+
|
|
60
|
+
expect(metadata.isValid).toBe(true);
|
|
61
|
+
expect(metadata.headers).toEqual(['timestamp']);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('should reject nested projections', () => {
|
|
65
|
+
const specification = GivenSpecificationFromInput(`(root: Root) {
|
|
66
|
+
successor: Successor [
|
|
67
|
+
successor->root: Root = root
|
|
68
|
+
]
|
|
69
|
+
} => {
|
|
70
|
+
nested = {
|
|
71
|
+
grandchild: Grandchild [
|
|
72
|
+
grandchild->successor: Successor = successor
|
|
73
|
+
]
|
|
74
|
+
} => {
|
|
75
|
+
name = grandchild.name
|
|
76
|
+
}
|
|
77
|
+
}`);
|
|
78
|
+
|
|
79
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
80
|
+
|
|
81
|
+
expect(metadata.isValid).toBe(false);
|
|
82
|
+
expect(metadata.errors.length).toBeGreaterThan(0);
|
|
83
|
+
expect(metadata.errors).toEqual([`Unsupported projection type of field 'nested' for CSV export. Only flat field projections are allowed.`]);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('should reject fact projections', () => {
|
|
87
|
+
const specification = GivenSpecificationFromInput(`(root: Root) {
|
|
88
|
+
successor: Successor [
|
|
89
|
+
successor->root: Root = root
|
|
90
|
+
]
|
|
91
|
+
} => {
|
|
92
|
+
successor = successor
|
|
93
|
+
}`);
|
|
94
|
+
|
|
95
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
96
|
+
|
|
97
|
+
expect(metadata.isValid).toBe(false);
|
|
98
|
+
expect(metadata.errors.length).toBeGreaterThan(0);
|
|
99
|
+
expect(metadata.errors).toEqual([`Unsupported projection type of field 'successor' for CSV export. Only flat field projections are allowed.`]);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('should reject non-composite top-level projections', () => {
|
|
103
|
+
const specification = GivenSpecificationFromInput(`(root: Root) {
|
|
104
|
+
successor: Successor [
|
|
105
|
+
successor->root: Root = root
|
|
106
|
+
]
|
|
107
|
+
} => successor.name`);
|
|
108
|
+
|
|
109
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
110
|
+
|
|
111
|
+
expect(metadata.isValid).toBe(false);
|
|
112
|
+
expect(metadata.errors).toContain('Specification projection must be composite for CSV export');
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('should reject specification without projection', () => {
|
|
116
|
+
const specification = GivenSpecificationFromInput(`(root: Root) {
|
|
117
|
+
successor: Successor [
|
|
118
|
+
successor->root: Root = root
|
|
119
|
+
]
|
|
120
|
+
}`);
|
|
121
|
+
|
|
122
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
123
|
+
|
|
124
|
+
expect(metadata.isValid).toBe(false);
|
|
125
|
+
expect(metadata.errors).toEqual(['Specification must have a projection for CSV export']);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('should handle mixed valid and invalid projections', () => {
|
|
129
|
+
const specification = GivenSpecificationFromInput(`(root: Root) {
|
|
130
|
+
successor: Successor [
|
|
131
|
+
successor->root: Root = root
|
|
132
|
+
]
|
|
133
|
+
} => {
|
|
134
|
+
name = successor.name
|
|
135
|
+
nested = {
|
|
136
|
+
grandchild: Grandchild [
|
|
137
|
+
grandchild->successor: Successor = successor
|
|
138
|
+
]
|
|
139
|
+
} => {
|
|
140
|
+
name = grandchild.name
|
|
141
|
+
}
|
|
142
|
+
count = successor.count
|
|
143
|
+
successor = successor
|
|
144
|
+
}`);
|
|
145
|
+
|
|
146
|
+
const metadata = validateSpecificationForCsv(specification);
|
|
147
|
+
|
|
148
|
+
expect(metadata.isValid).toBe(false);
|
|
149
|
+
expect(metadata.headers).toEqual(['name', 'count']); // Only valid field projections
|
|
150
|
+
expect(metadata.errors).toEqual([
|
|
151
|
+
`Unsupported projection type of field 'nested' for CSV export. Only flat field projections are allowed.`,
|
|
152
|
+
`Unsupported projection type of field 'successor' for CSV export. Only flat field projections are allowed.`
|
|
153
|
+
]);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
describe('extractValueByLabel', () => {
|
|
158
|
+
it('should extract direct property', () => {
|
|
159
|
+
const result = { name: 'Alice', age: 30 };
|
|
160
|
+
|
|
161
|
+
expect(extractValueByLabel(result, 'name')).toBe('Alice');
|
|
162
|
+
expect(extractValueByLabel(result, 'age')).toBe(30);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
it('should return null for missing properties', () => {
|
|
166
|
+
const result = { name: 'Alice' };
|
|
167
|
+
|
|
168
|
+
expect(extractValueByLabel(result, 'age')).toBeNull();
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it('should return null for null result', () => {
|
|
172
|
+
expect(extractValueByLabel(null, 'name')).toBeNull();
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('should return null for undefined result', () => {
|
|
176
|
+
expect(extractValueByLabel(undefined, 'name')).toBeNull();
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
describe('formatValueForCsv', () => {
|
|
181
|
+
it('should return empty string for null', () => {
|
|
182
|
+
expect(formatValueForCsv(null)).toBe('');
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it('should return empty string for undefined', () => {
|
|
186
|
+
expect(formatValueForCsv(undefined)).toBe('');
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it('should format Date as ISO string', () => {
|
|
190
|
+
const date = new Date('2024-01-15T10:30:00.000Z');
|
|
191
|
+
expect(formatValueForCsv(date)).toBe('2024-01-15T10:30:00.000Z');
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
it('should format boolean as string', () => {
|
|
195
|
+
expect(formatValueForCsv(true)).toBe('true');
|
|
196
|
+
expect(formatValueForCsv(false)).toBe('false');
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it('should convert primitives to string', () => {
|
|
200
|
+
expect(formatValueForCsv('hello')).toBe('hello');
|
|
201
|
+
expect(formatValueForCsv(42)).toBe('42');
|
|
202
|
+
expect(formatValueForCsv(3.14)).toBe('3.14');
|
|
203
|
+
});
|
|
204
|
+
});
|
|
205
|
+
});
|