jinaga-server 3.7.4 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/degrees-of-freedom/.openskills.json +7 -0
- package/.claude/skills/degrees-of-freedom/SKILL.md +18 -0
- package/.claude/skills/degrees-of-freedom/degrees-of-freedom-constitution.md +142 -0
- package/.claude/skills/night-shift-setup/.openskills.json +7 -0
- package/.claude/skills/night-shift-setup/SKILL.md +95 -0
- package/.claude/skills/night-shift-setup/configuring-a-repository.md +111 -0
- package/.claude/skills/night-shift-setup/creating-the-log.md +59 -0
- package/.claude/skills/night-shift-setup/creating-the-routines.md +80 -0
- package/.claude/skills/night-shift-setup/log-model.factual +971 -0
- package/.claude/skills/night-shift-worker/.openskills.json +7 -0
- package/.claude/skills/night-shift-worker/SKILL.md +199 -0
- package/.claude/skills/night-shift-worker/night-shift-log.md +72 -0
- package/.claude/skills/night-shift-worker/scripts/register-stack.sh +173 -0
- package/.claude/skills/night-shift-worker/stacking.md +57 -0
- package/.claude/skills/night-shift-worker/verifying-a-merged-fix.md +16 -0
- package/.claude/skills/refining-issues/.openskills.json +7 -0
- package/.claude/skills/refining-issues/SKILL.md +327 -0
- package/.claude/skills/refining-issues/references/worked-example.md +234 -0
- package/.github/workflows/main.yml +2 -2
- package/.night-shift/config.md +84 -0
- package/.night-shift/skills.md +17 -0
- package/README.md +21 -0
- package/dist/http/router.js +207 -80
- package/dist/http/router.js.map +1 -1
- package/dist/http/stream.d.ts +0 -1
- package/dist/http/stream.js +6 -17
- package/dist/http/stream.js.map +1 -1
- package/dist/jinaga-server.js +74 -7
- package/dist/jinaga-server.js.map +1 -1
- package/integration-test/jinaga-test/package-lock.json +4 -4
- package/integration-test/jinaga-test/package.json +1 -1
- package/package.json +2 -2
- package/src/http/router.ts +249 -99
- package/src/http/stream.ts +21 -36
- package/src/jinaga-server.ts +92 -4
- package/test/http/errorStatusMappingSpec.ts +351 -0
- package/test/http/streamLogVolumeSpec.ts +284 -0
- package/test/jinaga-server/poolConfigurationSpec.ts +147 -0
- package/test/jinaga-server/tracePoolSpec.ts +368 -0
package/src/http/router.ts
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
LoadResponse,
|
|
23
23
|
parseLoadMessage,
|
|
24
24
|
parseSaveMessage,
|
|
25
|
+
PredecessorNotResolvedError,
|
|
25
26
|
ProfileMessage,
|
|
26
27
|
ProjectedResult,
|
|
27
28
|
ReferencesByName,
|
|
@@ -40,6 +41,13 @@ import { outputReadResultsStreaming } from "./output-formatters";
|
|
|
40
41
|
import { arrayToResultStream, ResultStream } from "./result-stream";
|
|
41
42
|
import { Stream } from "./stream";
|
|
42
43
|
|
|
44
|
+
// A streaming response reports itself twice: one line when the stream opens
|
|
45
|
+
// and one when it ends, carrying the messages sent, how long it lasted, and
|
|
46
|
+
// what ended it. Issue #192: the per-message and per-lifecycle-step lines this
|
|
47
|
+
// replaces made the log grow with traffic rather than with the number of events
|
|
48
|
+
// worth knowing about, and `STREAM DONE`'s message count already said what the
|
|
49
|
+
// per-message lines said one at a time. A plain GET says nothing at all, since
|
|
50
|
+
// Trace.dependency already records it. Failures still log.
|
|
43
51
|
function getOrStream<U>(
|
|
44
52
|
getMethod: ((req: RequestUser, params: { [key: string]: string }, query: qs.ParsedQs) => Promise<U | null>),
|
|
45
53
|
streamMethod: ((req: RequestUser, params: { [key: string]: string }, query: qs.ParsedQs) => Promise<Stream<U> | null>)
|
|
@@ -48,75 +56,68 @@ function getOrStream<U>(
|
|
|
48
56
|
const connectionId = Math.random().toString(36).substring(2, 10);
|
|
49
57
|
const user = <RequestUser>(req as any).user;
|
|
50
58
|
const accept = req.headers["accept"];
|
|
51
|
-
|
|
52
|
-
console.log(`[HttpConnection:${connectionId}] New request - User: ${user?.id || 'anonymous'}, Accept: ${accept}, URL: ${req.url}`);
|
|
53
|
-
|
|
59
|
+
|
|
54
60
|
if (accept && accept.indexOf("application/x-jinaga-feed-stream") >= 0) {
|
|
55
|
-
console.log(`[HttpConnection:${connectionId}] STREAMING request detected`);
|
|
56
|
-
|
|
57
61
|
streamMethod(user, req.params, req.query)
|
|
58
62
|
.then(response => {
|
|
59
63
|
if (!response) {
|
|
60
|
-
|
|
61
|
-
res.sendStatus(404);
|
|
64
|
+
sendRouteMiss(res);
|
|
62
65
|
next();
|
|
63
66
|
}
|
|
64
67
|
else {
|
|
65
|
-
console.log(`[HttpConnection:${connectionId}] Setting up streaming response`);
|
|
66
|
-
|
|
67
68
|
res.type("application/x-jinaga-feed-stream");
|
|
68
69
|
res.set("Connection", "keep-alive");
|
|
69
70
|
res.set("Cache-Control", "no-cache");
|
|
70
71
|
res.set("Access-Control-Allow-Origin", "*");
|
|
71
72
|
res.flushHeaders();
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
|
|
74
|
+
const openedAt = Date.now();
|
|
75
|
+
console.log(`[HttpConnection:${connectionId}] Stream opened - User: ${user?.id || 'anonymous'}, URL: ${req.url}`);
|
|
76
|
+
|
|
75
77
|
let clientDisconnected = false;
|
|
76
|
-
let timeoutTriggered = false;
|
|
77
78
|
let streamClosed = false;
|
|
78
|
-
|
|
79
|
+
// What closed the stream from outside, where something
|
|
80
|
+
// other than the feed itself closed it. One value
|
|
81
|
+
// rather than a flag per cause, because a stream ends
|
|
82
|
+
// exactly once and only the first cause is the reason.
|
|
83
|
+
let closedBy: "client disconnect" | "timeout" | null = null;
|
|
84
|
+
|
|
79
85
|
// Client disconnect handler
|
|
80
86
|
req.on("close", () => {
|
|
81
|
-
console.log(`[HttpConnection:${connectionId}] CLIENT DISCONNECTED`);
|
|
82
87
|
clientDisconnected = true;
|
|
83
88
|
if (!streamClosed) {
|
|
84
|
-
|
|
85
|
-
response.close();
|
|
89
|
+
closedBy = "client disconnect";
|
|
86
90
|
streamClosed = true;
|
|
91
|
+
response.close();
|
|
87
92
|
}
|
|
88
93
|
});
|
|
89
|
-
|
|
94
|
+
|
|
90
95
|
// Timeout handler
|
|
91
96
|
const timeout = setTimeout(() => {
|
|
92
|
-
console.log(`[HttpConnection:${connectionId}] TIMEOUT TRIGGERED (5 minutes)`);
|
|
93
|
-
timeoutTriggered = true;
|
|
94
97
|
if (!streamClosed) {
|
|
95
|
-
|
|
96
|
-
response.close();
|
|
98
|
+
closedBy = "timeout";
|
|
97
99
|
streamClosed = true;
|
|
100
|
+
response.close();
|
|
98
101
|
}
|
|
99
102
|
}, 5 * 60 * 1000);
|
|
100
|
-
|
|
103
|
+
|
|
101
104
|
let messageCount = 0;
|
|
102
|
-
|
|
105
|
+
|
|
103
106
|
response
|
|
104
107
|
.next(data => {
|
|
105
108
|
messageCount++;
|
|
106
109
|
const writeStart = Date.now();
|
|
107
|
-
|
|
110
|
+
|
|
108
111
|
if (clientDisconnected) {
|
|
109
112
|
console.warn(`[HttpConnection:${connectionId}] Attempted to write to disconnected client - Message #${messageCount}`);
|
|
110
113
|
return;
|
|
111
114
|
}
|
|
112
|
-
|
|
115
|
+
|
|
113
116
|
try {
|
|
114
117
|
const jsonData = JSON.stringify(data) + "\n\n";
|
|
115
118
|
res.write(jsonData);
|
|
116
119
|
const writeDuration = Date.now() - writeStart;
|
|
117
|
-
|
|
118
|
-
console.log(`[HttpConnection:${connectionId}] Message #${messageCount} sent - Size: ${jsonData.length} bytes, Duration: ${writeDuration}ms`);
|
|
119
|
-
|
|
120
|
+
|
|
120
121
|
if (writeDuration > 50) {
|
|
121
122
|
console.warn(`[HttpConnection:${connectionId}] SLOW write operation - Duration: ${writeDuration}ms`);
|
|
122
123
|
}
|
|
@@ -125,24 +126,19 @@ function getOrStream<U>(
|
|
|
125
126
|
}
|
|
126
127
|
})
|
|
127
128
|
.done(() => {
|
|
128
|
-
console.log(`[HttpConnection:${connectionId}] STREAM DONE - Messages sent: ${messageCount}, Client disconnected: ${clientDisconnected}, Timeout triggered: ${timeoutTriggered}`);
|
|
129
|
-
|
|
130
129
|
clearTimeout(timeout);
|
|
131
130
|
streamClosed = true;
|
|
132
|
-
|
|
131
|
+
|
|
132
|
+
console.log(`[HttpConnection:${connectionId}] Stream ended - Messages sent: ${messageCount}, Duration: ${Date.now() - openedAt}ms, Reason: ${closedBy ?? "done"}`);
|
|
133
|
+
|
|
133
134
|
try {
|
|
134
135
|
if (!clientDisconnected) {
|
|
135
|
-
console.log(`[HttpConnection:${connectionId}] Ending socket connection`);
|
|
136
136
|
res.socket?.end();
|
|
137
|
-
} else {
|
|
138
|
-
console.log(`[HttpConnection:${connectionId}] Client already disconnected - skipping socket end`);
|
|
139
137
|
}
|
|
140
138
|
} catch (error) {
|
|
141
139
|
console.error(`[HttpConnection:${connectionId}] ERROR ending socket: ${error}`);
|
|
142
140
|
}
|
|
143
141
|
});
|
|
144
|
-
|
|
145
|
-
console.log(`[HttpConnection:${connectionId}] Stream handlers configured`);
|
|
146
142
|
}
|
|
147
143
|
})
|
|
148
144
|
.catch(error => {
|
|
@@ -151,16 +147,13 @@ function getOrStream<U>(
|
|
|
151
147
|
});
|
|
152
148
|
}
|
|
153
149
|
else {
|
|
154
|
-
console.log(`[HttpConnection:${connectionId}] Regular GET request`);
|
|
155
150
|
getMethod(user, req.params, req.query)
|
|
156
151
|
.then(response => {
|
|
157
152
|
if (!response) {
|
|
158
|
-
|
|
159
|
-
res.sendStatus(404);
|
|
153
|
+
sendRouteMiss(res);
|
|
160
154
|
next();
|
|
161
155
|
}
|
|
162
156
|
else {
|
|
163
|
-
console.log(`[HttpConnection:${connectionId}] Sending JSON response`);
|
|
164
157
|
res.type("json");
|
|
165
158
|
res.send(JSON.stringify(response));
|
|
166
159
|
next();
|
|
@@ -199,9 +192,14 @@ function post<T, U>(
|
|
|
199
192
|
): Handler {
|
|
200
193
|
return (req, res, next) => {
|
|
201
194
|
const user = <RequestUser>(req as any).user;
|
|
202
|
-
const
|
|
195
|
+
const parsed = parseRequestInput(() => parse(req.body), req, res, next);
|
|
196
|
+
if (!parsed.ok) {
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
const message = parsed.value;
|
|
203
200
|
if (!message) {
|
|
204
|
-
|
|
201
|
+
handleError(new Invalid(`${MISSING_BODY} ${describeContentType(req)}`), req, res, next);
|
|
202
|
+
return;
|
|
205
203
|
}
|
|
206
204
|
method(user, message, req.params)
|
|
207
205
|
.then(response => {
|
|
@@ -224,8 +222,11 @@ function postCreate<T>(
|
|
|
224
222
|
): Handler {
|
|
225
223
|
return (req, res, next) => {
|
|
226
224
|
const user = <RequestUser>(req as any).user;
|
|
227
|
-
const
|
|
228
|
-
|
|
225
|
+
const parsed = parseRequestInput(() => parse(req), req, res, next);
|
|
226
|
+
if (!parsed.ok) {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
method(user, parsed.value)
|
|
229
230
|
.then(_ => {
|
|
230
231
|
res.sendStatus(201);
|
|
231
232
|
next();
|
|
@@ -237,19 +238,16 @@ function postCreate<T>(
|
|
|
237
238
|
function postStringCreate(method: (user: RequestUser, message: string) => Promise<void>): Handler {
|
|
238
239
|
return (req, res, next) => {
|
|
239
240
|
const user = <RequestUser>(req as any).user;
|
|
240
|
-
const
|
|
241
|
-
if (!
|
|
242
|
-
|
|
243
|
-
res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
|
|
244
|
-
}
|
|
245
|
-
else {
|
|
246
|
-
method(user, input)
|
|
247
|
-
.then(_ => {
|
|
248
|
-
res.sendStatus(201);
|
|
249
|
-
next();
|
|
250
|
-
})
|
|
251
|
-
.catch(error => handleError(error, req, res, next));
|
|
241
|
+
const parsed = parseRequestInput(() => parseTextBody(req), req, res, next);
|
|
242
|
+
if (!parsed.ok) {
|
|
243
|
+
return;
|
|
252
244
|
}
|
|
245
|
+
method(user, parsed.value)
|
|
246
|
+
.then(_ => {
|
|
247
|
+
res.sendStatus(201);
|
|
248
|
+
next();
|
|
249
|
+
})
|
|
250
|
+
.catch(error => handleError(error, req, res, next));
|
|
253
251
|
};
|
|
254
252
|
}
|
|
255
253
|
|
|
@@ -261,29 +259,27 @@ function postReadWithStreaming(
|
|
|
261
259
|
): Handler {
|
|
262
260
|
return (req, res, next) => {
|
|
263
261
|
const user = <RequestUser>(req as any).user;
|
|
264
|
-
const
|
|
265
|
-
if (!
|
|
266
|
-
|
|
267
|
-
res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
|
|
262
|
+
const parsed = parseRequestInput(() => parseTextBody(req), req, res, next);
|
|
263
|
+
if (!parsed.ok) {
|
|
264
|
+
return;
|
|
268
265
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
method(user, input, acceptType)
|
|
281
|
-
.then(({resultStream, csvMetadata}) => {
|
|
282
|
-
outputReadResults(resultStream, res, acceptType, csvMetadata);
|
|
283
|
-
next();
|
|
284
|
-
})
|
|
285
|
-
.catch(error => handleError(error, req, res, next));
|
|
266
|
+
|
|
267
|
+
// Check if Accept header explicitly prefers a specific format
|
|
268
|
+
const acceptHeader = req.get('Accept');
|
|
269
|
+
let acceptType: string = 'text/plain'; // Default for backward compatibility
|
|
270
|
+
|
|
271
|
+
if (acceptHeader && acceptHeader !== '*/*') {
|
|
272
|
+
// Only use req.accepts() when there's a specific preference
|
|
273
|
+
const preferredType = req.accepts(['text/csv', 'application/x-ndjson', 'application/json', 'text/plain']);
|
|
274
|
+
acceptType = preferredType ? String(preferredType) : 'text/plain';
|
|
286
275
|
}
|
|
276
|
+
|
|
277
|
+
method(user, parsed.value, acceptType)
|
|
278
|
+
.then(({resultStream, csvMetadata}) => {
|
|
279
|
+
outputReadResults(resultStream, res, acceptType, csvMetadata);
|
|
280
|
+
next();
|
|
281
|
+
})
|
|
282
|
+
.catch(error => handleError(error, req, res, next));
|
|
287
283
|
};
|
|
288
284
|
}
|
|
289
285
|
|
|
@@ -313,7 +309,7 @@ function inputSaveMessage(req: Request): GraphSource {
|
|
|
313
309
|
read: async (onEnvelopes) => {
|
|
314
310
|
const message = parseSaveMessage(req.body);
|
|
315
311
|
if (!message) {
|
|
316
|
-
throw new
|
|
312
|
+
throw new Invalid(MISSING_JSON_BODY);
|
|
317
313
|
}
|
|
318
314
|
await onEnvelopes(message.facts.map(fact => ({
|
|
319
315
|
fact: fact,
|
|
@@ -358,7 +354,9 @@ function outputReadResults(
|
|
|
358
354
|
.catch(error => {
|
|
359
355
|
console.error('Error in outputReadResults:', error);
|
|
360
356
|
if (!res.headersSent) {
|
|
361
|
-
res.
|
|
357
|
+
// Without an explicit type, res.send of a string makes Express
|
|
358
|
+
// infer text/html and label an error body as markup.
|
|
359
|
+
res.type("text").status(500).send('Internal server error');
|
|
362
360
|
}
|
|
363
361
|
});
|
|
364
362
|
}
|
|
@@ -550,11 +548,19 @@ export class HttpRouter {
|
|
|
550
548
|
}
|
|
551
549
|
|
|
552
550
|
const userIdentity = serializeUserIdentity(user);
|
|
553
|
-
|
|
551
|
+
const envelopes: FactEnvelope[] = factRecords
|
|
554
552
|
.map(fact => ({
|
|
555
553
|
fact: fact,
|
|
556
554
|
signatures: []
|
|
557
|
-
}))
|
|
555
|
+
}));
|
|
556
|
+
try {
|
|
557
|
+
await this.authorization.save(userIdentity, envelopes);
|
|
558
|
+
} catch (error) {
|
|
559
|
+
// /write reaches the same authorization path as /save, so it
|
|
560
|
+
// needs the same translation of predecessor-resolution
|
|
561
|
+
// failures into a diagnosable 4xx (issue #182 finding 2).
|
|
562
|
+
throw toSaveAuthorizationError(error, envelopes);
|
|
563
|
+
}
|
|
558
564
|
});
|
|
559
565
|
}
|
|
560
566
|
|
|
@@ -988,34 +994,174 @@ export class FeedNotFound extends Error {
|
|
|
988
994
|
}
|
|
989
995
|
}
|
|
990
996
|
|
|
997
|
+
// Both message shapes below are matched by prose, because jinaga reports a
|
|
998
|
+
// missing predecessor as a plain Error. That is fragile: an upstream wording
|
|
999
|
+
// change silently turns these back into 500s (issue #182 finding 5). The
|
|
1000
|
+
// end-to-end cases in test/http/errorStatusMappingSpec.ts drive the real
|
|
1001
|
+
// AuthorizationEngine so that drift fails the build.
|
|
1002
|
+
//
|
|
1003
|
+
// jinaga 6.11.3 landed the typed errors of jinaga.js#234, so the topological
|
|
1004
|
+
// sort's failure is now an instanceof check below. One prose pattern remains:
|
|
1005
|
+
// specification-runner still reports its missing fact as a plain Error.
|
|
1006
|
+
|
|
991
1007
|
// Authorization rule evaluation reports a missing predecessor as a plain
|
|
992
|
-
// Error of this shape (
|
|
993
|
-
//
|
|
994
|
-
//
|
|
995
|
-
//
|
|
1008
|
+
// Error of this shape (specification-runner.ts in jinaga, still untyped as of
|
|
1009
|
+
// 6.11.3). It surfaces when the client's save request didn't include the full
|
|
1010
|
+
// closure of facts a rule needs to walk. Map it to a diagnosable 4xx instead
|
|
1011
|
+
// of an opaque 500 (issue #175).
|
|
996
1012
|
const FACT_NOT_DEFINED_PATTERN = /^The fact (\S+):(\S+) is not defined\.$/;
|
|
997
1013
|
|
|
998
1014
|
function toSaveAuthorizationError(error: any, envelopes: FactEnvelope[]): any {
|
|
999
|
-
if (error instanceof Error) {
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1015
|
+
if (!(error instanceof Error)) {
|
|
1016
|
+
return error;
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
const describeBatch = () => {
|
|
1020
|
+
const factTypesInBatch = Array.from(new Set(envelopes.map(e => e.fact.type))).join(", ");
|
|
1021
|
+
return `${envelopes.length} fact(s) in the request (types: ${factTypesInBatch})`;
|
|
1022
|
+
};
|
|
1023
|
+
|
|
1024
|
+
// The batch named a predecessor that is in neither the batch nor storage,
|
|
1025
|
+
// so the topological sort could never reach the facts that depend on it.
|
|
1026
|
+
// A client data problem, not a server fault (issue #182 finding 1).
|
|
1027
|
+
if (error instanceof PredecessorNotResolvedError) {
|
|
1028
|
+
const missing = error.missingPredecessors.map(r => `${r.type}:${r.hash}`);
|
|
1029
|
+
const unresolved = error.unresolvedFacts.map(r => `${r.type}:${r.hash}`);
|
|
1030
|
+
Trace.warn(
|
|
1031
|
+
`Save authorization could not resolve ${missing.length} predecessor(s) [${missing.join(", ")}] ` +
|
|
1032
|
+
`needed by ${unresolved.length} fact(s) [${unresolved.join(", ")}] among ${describeBatch()}.`
|
|
1033
|
+
);
|
|
1034
|
+
const advice = `Send the full closure of predecessors together, or commit the predecessors first.`;
|
|
1035
|
+
if (missing.length === 1) {
|
|
1036
|
+
return new Invalid(
|
|
1037
|
+
`The fact ${missing[0]} is named as a predecessor, but it was not found in storage ` +
|
|
1038
|
+
`and was not included in the request. ${advice}`
|
|
1007
1039
|
);
|
|
1040
|
+
}
|
|
1041
|
+
if (missing.length > 1) {
|
|
1008
1042
|
return new Invalid(
|
|
1009
|
-
|
|
1010
|
-
`
|
|
1043
|
+
`${missing.length} facts are named as predecessors, but they were not found in storage ` +
|
|
1044
|
+
`and were not included in the request: ${missing.join(", ")}. ${advice}`
|
|
1011
1045
|
);
|
|
1012
1046
|
}
|
|
1047
|
+
// The engine could not attribute the failure to specific references.
|
|
1048
|
+
return new Invalid(
|
|
1049
|
+
`${unresolved.length} fact(s) could not be authorized because one or more predecessors ` +
|
|
1050
|
+
`were not found in storage and were not included in the request. ${advice}`
|
|
1051
|
+
);
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
const notDefined = FACT_NOT_DEFINED_PATTERN.exec(error.message);
|
|
1055
|
+
if (notDefined) {
|
|
1056
|
+
const [, factType, factHash] = notDefined;
|
|
1057
|
+
Trace.warn(
|
|
1058
|
+
`Save authorization could not resolve predecessor ${factType}:${factHash} ` +
|
|
1059
|
+
`among ${describeBatch()}.`
|
|
1060
|
+
);
|
|
1061
|
+
return new Invalid(
|
|
1062
|
+
`The fact ${factType}:${factHash} is required to authorize this save but was not included in the request. ` +
|
|
1063
|
+
`Ensure the full closure of predecessors needed by your authorization rules is sent together.`
|
|
1064
|
+
);
|
|
1013
1065
|
}
|
|
1066
|
+
|
|
1014
1067
|
return error;
|
|
1015
1068
|
}
|
|
1016
1069
|
|
|
1070
|
+
// post() serves both a JSON route (/load) and a text route (/feeds), so its
|
|
1071
|
+
// message names no particular parser; the single-content-type entry points
|
|
1072
|
+
// keep the specific hint they have always given.
|
|
1073
|
+
const MISSING_BODY = 'The request body is empty or could not be read. Ensure that the body parser for this Content-Type is registered.';
|
|
1074
|
+
const MISSING_JSON_BODY = 'Ensure that you have called app.use(express.json()).';
|
|
1075
|
+
const MISSING_TEXT_BODY = 'Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).';
|
|
1076
|
+
|
|
1077
|
+
// A request body that fails to parse is a client error. Both this module's
|
|
1078
|
+
// parseString and jinaga's message parsers signal that by throwing
|
|
1079
|
+
// synchronously from the handler, where Express hands it to its default error
|
|
1080
|
+
// handler and reports 500. Classify parse failures as Invalid so handleError
|
|
1081
|
+
// renders a 400 (issue #182 finding 3).
|
|
1082
|
+
function toInvalidInput(error: any): Invalid {
|
|
1083
|
+
if (error instanceof Invalid) {
|
|
1084
|
+
return error;
|
|
1085
|
+
}
|
|
1086
|
+
return new Invalid(error instanceof Error ? error.message : String(error));
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
// /read and /write accept text/plain only, so every unreadable body there has
|
|
1090
|
+
// the same cause and deserves the same actionable hint. Checking req.body here
|
|
1091
|
+
// rather than deferring to parseString keeps a missing body parser — the common
|
|
1092
|
+
// misconfiguration — from being reported with parseString's generic
|
|
1093
|
+
// content-type message.
|
|
1094
|
+
function parseTextBody(req: Request): string {
|
|
1095
|
+
const body = req.body;
|
|
1096
|
+
if (typeof body !== 'string' || !body) {
|
|
1097
|
+
throw new Invalid(`${MISSING_TEXT_BODY} ${describeContentType(req)}`);
|
|
1098
|
+
}
|
|
1099
|
+
return body;
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
// Naming what arrived turns "check your setup" into something the caller can
|
|
1103
|
+
// act on without guessing which end is misconfigured. The value is echoed from
|
|
1104
|
+
// the request, so cap it: a real content type is short, and the body should
|
|
1105
|
+
// not grow with whatever the caller chose to send.
|
|
1106
|
+
const MAX_ECHOED_CONTENT_TYPE = 80;
|
|
1107
|
+
|
|
1108
|
+
function describeContentType(req: Request): string {
|
|
1109
|
+
const contentType = req.get('Content-Type');
|
|
1110
|
+
if (!contentType) {
|
|
1111
|
+
return `No Content-Type header was received.`;
|
|
1112
|
+
}
|
|
1113
|
+
const echoed = contentType.length > MAX_ECHOED_CONTENT_TYPE
|
|
1114
|
+
? `${contentType.substring(0, MAX_ECHOED_CONTENT_TYPE)}…`
|
|
1115
|
+
: contentType;
|
|
1116
|
+
return `Received Content-Type: ${echoed}.`;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
function parseRequestInput<T>(
|
|
1120
|
+
parse: () => T,
|
|
1121
|
+
req: Request,
|
|
1122
|
+
res: Response,
|
|
1123
|
+
next: NextFunction
|
|
1124
|
+
): { ok: true, value: T } | { ok: false } {
|
|
1125
|
+
try {
|
|
1126
|
+
return { ok: true, value: parse() };
|
|
1127
|
+
} catch (error) {
|
|
1128
|
+
handleError(toInvalidInput(error), req, res, next);
|
|
1129
|
+
return { ok: false };
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// A route miss (no :hash in the URL) is a 404 just like a feed-cache miss,
|
|
1134
|
+
// but for a different reason. Send a machine-readable body so a client can
|
|
1135
|
+
// always tell "wrong URL" from "re-subscribe via POST /feeds" (issue #182
|
|
1136
|
+
// finding 6); handleError sends "feed_not_found" for the latter.
|
|
1137
|
+
function sendRouteMiss(res: Response) {
|
|
1138
|
+
res.type("text");
|
|
1139
|
+
res.status(404).send("not_found");
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
// These bodies are diagnostics, not markup, but they quote client-supplied
|
|
1143
|
+
// text: a Content-Type header, a fact type named in a parse failure. The
|
|
1144
|
+
// response is text/plain and marked nosniff, yet that is the content type
|
|
1145
|
+
// saying so — neutralize the characters that could start markup here, at the
|
|
1146
|
+
// one place every error body is written, so the guarantee does not rest on a
|
|
1147
|
+
// header a proxy or a later edit could change. Quotes are left alone: they
|
|
1148
|
+
// only matter inside a tag, which escaping '<' already prevents, and messages
|
|
1149
|
+
// quote identifiers routinely.
|
|
1150
|
+
const HTML_META_CHARACTERS: { [character: string]: string } = {
|
|
1151
|
+
"&": "&",
|
|
1152
|
+
"<": "<",
|
|
1153
|
+
">": ">"
|
|
1154
|
+
};
|
|
1155
|
+
|
|
1156
|
+
function sanitizeErrorBody(message: string): string {
|
|
1157
|
+
return message.replace(/[&<>]/g, character => HTML_META_CHARACTERS[character]);
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1017
1160
|
function handleError(error: any, req: Request, res: Response, next: NextFunction) {
|
|
1018
1161
|
const requestPath = req.path;
|
|
1162
|
+
// Error bodies are plain text built partly from client-supplied input, so
|
|
1163
|
+
// stop a browser from sniffing one as markup.
|
|
1164
|
+
res.set("X-Content-Type-Options", "nosniff");
|
|
1019
1165
|
if (error instanceof FeedNotFound) {
|
|
1020
1166
|
Trace.warn(`Feed not found: ${error.feedHash} (Path: ${requestPath})`);
|
|
1021
1167
|
res.type("text");
|
|
@@ -1023,15 +1169,19 @@ function handleError(error: any, req: Request, res: Response, next: NextFunction
|
|
|
1023
1169
|
} else if (error instanceof Forbidden) {
|
|
1024
1170
|
Trace.warn(`Forbidden: ${error.message} (Path: ${requestPath})`);
|
|
1025
1171
|
res.type("text");
|
|
1026
|
-
res.status(403).send(error.message);
|
|
1172
|
+
res.status(403).send(sanitizeErrorBody(error.message));
|
|
1027
1173
|
} else if (error instanceof Invalid) {
|
|
1028
1174
|
Trace.warn(`Invalid: ${error.message} (Path: ${requestPath})`);
|
|
1029
1175
|
res.type("text");
|
|
1030
|
-
res.status(400).send(error.message);
|
|
1176
|
+
res.status(400).send(sanitizeErrorBody(error.message));
|
|
1031
1177
|
} else {
|
|
1178
|
+
// Classification above is an allow-list, so anything reaching here is
|
|
1179
|
+
// unclassified — including genuine internal failures whose message can
|
|
1180
|
+
// name internal hosts or credentials. Keep the detail in the trace and
|
|
1181
|
+
// send a generic body to the caller (issue #182 finding 4).
|
|
1032
1182
|
Trace.error(`Error: ${error.message} (Path: ${requestPath})`);
|
|
1033
1183
|
res.type("text");
|
|
1034
|
-
res.status(500).send(error
|
|
1184
|
+
res.status(500).send("Internal server error");
|
|
1035
1185
|
}
|
|
1036
1186
|
next();
|
|
1037
1187
|
}
|
package/src/http/stream.ts
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
|
+
// This class logs nothing on its success path. A stream's lifetime is reported
|
|
2
|
+
// once by the request that opened it (see getOrStream in router.ts), and a line
|
|
3
|
+
// per feed, per replayed item, or per done handler says the same thing again
|
|
4
|
+
// once per unit of traffic. Issue #192: those per-item lines were 11.9% of a
|
|
5
|
+
// 2.8-million-line production log. Failures still log, because a failed handler
|
|
6
|
+
// is an event an operator cannot reconstruct from the summary.
|
|
1
7
|
export class Stream<T> {
|
|
2
8
|
private queue: T[] = [];
|
|
3
9
|
private handlers: ((data: T) => void)[] = [];
|
|
4
10
|
private doneHandlers: (() => void)[] = [];
|
|
5
11
|
private closed = false;
|
|
6
12
|
private readonly streamId: string;
|
|
7
|
-
private feedCount = 0;
|
|
8
13
|
|
|
9
14
|
constructor() {
|
|
10
15
|
this.streamId = Math.random().toString(36).substring(2, 10);
|
|
11
|
-
console.log(`[Stream:${this.streamId}] Created new stream`);
|
|
12
16
|
}
|
|
13
17
|
|
|
14
18
|
next(handler: (data: T) => void): Stream<T> {
|
|
@@ -16,21 +20,18 @@ export class Stream<T> {
|
|
|
16
20
|
console.warn(`[Stream:${this.streamId}] Attempted to add handler to closed stream`);
|
|
17
21
|
return this;
|
|
18
22
|
}
|
|
19
|
-
|
|
23
|
+
|
|
20
24
|
this.handlers.push(handler);
|
|
21
|
-
|
|
22
|
-
console.log(`[Stream:${this.streamId}] Added handler - Total handlers: ${this.handlers.length}, Queued items: ${queueLength}`);
|
|
23
|
-
|
|
25
|
+
|
|
24
26
|
// Replay queued data to new handler
|
|
25
27
|
this.queue.forEach((data, index) => {
|
|
26
28
|
try {
|
|
27
29
|
handler(data);
|
|
28
|
-
console.log(`[Stream:${this.streamId}] Replayed queued item ${index + 1}/${queueLength} to new handler`);
|
|
29
30
|
} catch (error) {
|
|
30
31
|
console.error(`[Stream:${this.streamId}] Error replaying queued item ${index + 1}: ${error}`);
|
|
31
32
|
}
|
|
32
33
|
});
|
|
33
|
-
|
|
34
|
+
|
|
34
35
|
return this;
|
|
35
36
|
}
|
|
36
37
|
|
|
@@ -44,9 +45,8 @@ export class Stream<T> {
|
|
|
44
45
|
}
|
|
45
46
|
return this;
|
|
46
47
|
}
|
|
47
|
-
|
|
48
|
+
|
|
48
49
|
this.doneHandlers.push(handler);
|
|
49
|
-
console.log(`[Stream:${this.streamId}] Added done handler - Total done handlers: ${this.doneHandlers.length}`);
|
|
50
50
|
return this;
|
|
51
51
|
}
|
|
52
52
|
|
|
@@ -55,70 +55,55 @@ export class Stream<T> {
|
|
|
55
55
|
console.warn(`[Stream:${this.streamId}] Attempted to close already closed stream`);
|
|
56
56
|
return;
|
|
57
57
|
}
|
|
58
|
-
|
|
59
|
-
console.log(`[Stream:${this.streamId}] CLOSING stream - Handlers: ${this.handlers.length}, Done handlers: ${this.doneHandlers.length}, Queue: ${this.queue.length}, Feed count: ${this.feedCount}`);
|
|
60
|
-
|
|
58
|
+
|
|
61
59
|
this.closed = true;
|
|
62
|
-
|
|
60
|
+
|
|
63
61
|
// Execute done handlers
|
|
64
|
-
const doneHandlerCount = this.doneHandlers.length;
|
|
65
62
|
this.doneHandlers.forEach((handler, index) => {
|
|
66
63
|
try {
|
|
67
64
|
handler();
|
|
68
|
-
console.log(`[Stream:${this.streamId}] Executed done handler ${index + 1}/${doneHandlerCount}`);
|
|
69
65
|
} catch (error) {
|
|
70
66
|
console.error(`[Stream:${this.streamId}] Error in done handler ${index + 1}: ${error}`);
|
|
71
67
|
}
|
|
72
68
|
});
|
|
73
|
-
|
|
69
|
+
|
|
74
70
|
// Clear all references
|
|
75
71
|
this.queue = [];
|
|
76
72
|
this.handlers = [];
|
|
77
73
|
this.doneHandlers = [];
|
|
78
|
-
|
|
79
|
-
console.log(`[Stream:${this.streamId}] Stream closed and cleaned up`);
|
|
80
74
|
}
|
|
81
75
|
|
|
82
76
|
feed(data: any): void {
|
|
83
77
|
const feedStart = Date.now();
|
|
84
|
-
|
|
78
|
+
|
|
85
79
|
if (this.closed) {
|
|
86
80
|
console.warn(`[Stream:${this.streamId}] Attempted to feed data to closed stream - Data type: ${typeof data}`);
|
|
87
81
|
return;
|
|
88
82
|
}
|
|
89
|
-
|
|
90
|
-
this.feedCount++;
|
|
83
|
+
|
|
91
84
|
const handlerCount = this.handlers.length;
|
|
92
|
-
|
|
93
|
-
console.log(`[Stream:${this.streamId}] FEEDING data - Feed #${this.feedCount}, Handlers: ${handlerCount}, Queue size before: ${this.queue.length}`);
|
|
94
|
-
|
|
85
|
+
|
|
95
86
|
this.queue.push(data);
|
|
96
|
-
|
|
87
|
+
|
|
97
88
|
// Notify all handlers
|
|
98
|
-
let successCount = 0;
|
|
99
|
-
let errorCount = 0;
|
|
100
|
-
|
|
101
89
|
this.handlers.forEach((handler, index) => {
|
|
102
90
|
try {
|
|
103
91
|
const handlerStart = Date.now();
|
|
104
92
|
handler(data);
|
|
105
93
|
const handlerDuration = Date.now() - handlerStart;
|
|
106
|
-
|
|
107
|
-
|
|
94
|
+
|
|
108
95
|
if (handlerDuration > 50) {
|
|
109
96
|
console.warn(`[Stream:${this.streamId}] SLOW handler - Handler ${index + 1}/${handlerCount}, Duration: ${handlerDuration}ms`);
|
|
110
97
|
}
|
|
111
98
|
} catch (error) {
|
|
112
|
-
errorCount++;
|
|
113
99
|
console.error(`[Stream:${this.streamId}] ERROR in handler ${index + 1}/${handlerCount}: ${error}`);
|
|
114
100
|
}
|
|
115
101
|
});
|
|
116
|
-
|
|
102
|
+
|
|
117
103
|
const totalDuration = Date.now() - feedStart;
|
|
118
|
-
|
|
119
|
-
|
|
104
|
+
|
|
120
105
|
if (totalDuration > 100) {
|
|
121
106
|
console.warn(`[Stream:${this.streamId}] SLOW feed operation - Duration: ${totalDuration}ms`);
|
|
122
107
|
}
|
|
123
108
|
}
|
|
124
|
-
}
|
|
109
|
+
}
|