jinaga-server 3.7.4 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/.claude/skills/degrees-of-freedom/.openskills.json +7 -0
  2. package/.claude/skills/degrees-of-freedom/SKILL.md +18 -0
  3. package/.claude/skills/degrees-of-freedom/degrees-of-freedom-constitution.md +142 -0
  4. package/.claude/skills/night-shift-setup/.openskills.json +7 -0
  5. package/.claude/skills/night-shift-setup/SKILL.md +95 -0
  6. package/.claude/skills/night-shift-setup/configuring-a-repository.md +111 -0
  7. package/.claude/skills/night-shift-setup/creating-the-log.md +59 -0
  8. package/.claude/skills/night-shift-setup/creating-the-routines.md +80 -0
  9. package/.claude/skills/night-shift-setup/log-model.factual +971 -0
  10. package/.claude/skills/night-shift-worker/.openskills.json +7 -0
  11. package/.claude/skills/night-shift-worker/SKILL.md +199 -0
  12. package/.claude/skills/night-shift-worker/night-shift-log.md +72 -0
  13. package/.claude/skills/night-shift-worker/scripts/register-stack.sh +173 -0
  14. package/.claude/skills/night-shift-worker/stacking.md +57 -0
  15. package/.claude/skills/night-shift-worker/verifying-a-merged-fix.md +16 -0
  16. package/.claude/skills/refining-issues/.openskills.json +7 -0
  17. package/.claude/skills/refining-issues/SKILL.md +327 -0
  18. package/.claude/skills/refining-issues/references/worked-example.md +234 -0
  19. package/.github/workflows/main.yml +2 -2
  20. package/.night-shift/config.md +84 -0
  21. package/.night-shift/skills.md +17 -0
  22. package/README.md +21 -0
  23. package/dist/http/router.js +207 -80
  24. package/dist/http/router.js.map +1 -1
  25. package/dist/http/stream.d.ts +0 -1
  26. package/dist/http/stream.js +6 -17
  27. package/dist/http/stream.js.map +1 -1
  28. package/dist/jinaga-server.js +74 -7
  29. package/dist/jinaga-server.js.map +1 -1
  30. package/integration-test/jinaga-test/package-lock.json +4 -4
  31. package/integration-test/jinaga-test/package.json +1 -1
  32. package/package.json +2 -2
  33. package/src/http/router.ts +249 -99
  34. package/src/http/stream.ts +21 -36
  35. package/src/jinaga-server.ts +92 -4
  36. package/test/http/errorStatusMappingSpec.ts +351 -0
  37. package/test/http/streamLogVolumeSpec.ts +284 -0
  38. package/test/jinaga-server/poolConfigurationSpec.ts +147 -0
  39. package/test/jinaga-server/tracePoolSpec.ts +368 -0
@@ -22,6 +22,7 @@ import {
22
22
  LoadResponse,
23
23
  parseLoadMessage,
24
24
  parseSaveMessage,
25
+ PredecessorNotResolvedError,
25
26
  ProfileMessage,
26
27
  ProjectedResult,
27
28
  ReferencesByName,
@@ -40,6 +41,13 @@ import { outputReadResultsStreaming } from "./output-formatters";
40
41
  import { arrayToResultStream, ResultStream } from "./result-stream";
41
42
  import { Stream } from "./stream";
42
43
 
44
+ // A streaming response reports itself twice: one line when the stream opens
45
+ // and one when it ends, carrying the messages sent, how long it lasted, and
46
+ // what ended it. Issue #192: the per-message and per-lifecycle-step lines this
47
+ // replaces made the log grow with traffic rather than with the number of events
48
+ // worth knowing about, and `STREAM DONE`'s message count already said what the
49
+ // per-message lines said one at a time. A plain GET says nothing at all, since
50
+ // Trace.dependency already records it. Failures still log.
43
51
  function getOrStream<U>(
44
52
  getMethod: ((req: RequestUser, params: { [key: string]: string }, query: qs.ParsedQs) => Promise<U | null>),
45
53
  streamMethod: ((req: RequestUser, params: { [key: string]: string }, query: qs.ParsedQs) => Promise<Stream<U> | null>)
@@ -48,75 +56,68 @@ function getOrStream<U>(
48
56
  const connectionId = Math.random().toString(36).substring(2, 10);
49
57
  const user = <RequestUser>(req as any).user;
50
58
  const accept = req.headers["accept"];
51
-
52
- console.log(`[HttpConnection:${connectionId}] New request - User: ${user?.id || 'anonymous'}, Accept: ${accept}, URL: ${req.url}`);
53
-
59
+
54
60
  if (accept && accept.indexOf("application/x-jinaga-feed-stream") >= 0) {
55
- console.log(`[HttpConnection:${connectionId}] STREAMING request detected`);
56
-
57
61
  streamMethod(user, req.params, req.query)
58
62
  .then(response => {
59
63
  if (!response) {
60
- console.log(`[HttpConnection:${connectionId}] Stream method returned null - sending 404`);
61
- res.sendStatus(404);
64
+ sendRouteMiss(res);
62
65
  next();
63
66
  }
64
67
  else {
65
- console.log(`[HttpConnection:${connectionId}] Setting up streaming response`);
66
-
67
68
  res.type("application/x-jinaga-feed-stream");
68
69
  res.set("Connection", "keep-alive");
69
70
  res.set("Cache-Control", "no-cache");
70
71
  res.set("Access-Control-Allow-Origin", "*");
71
72
  res.flushHeaders();
72
-
73
- console.log(`[HttpConnection:${connectionId}] Headers flushed - setting up event handlers`);
74
-
73
+
74
+ const openedAt = Date.now();
75
+ console.log(`[HttpConnection:${connectionId}] Stream opened - User: ${user?.id || 'anonymous'}, URL: ${req.url}`);
76
+
75
77
  let clientDisconnected = false;
76
- let timeoutTriggered = false;
77
78
  let streamClosed = false;
78
-
79
+ // What closed the stream from outside, where something
80
+ // other than the feed itself closed it. One value
81
+ // rather than a flag per cause, because a stream ends
82
+ // exactly once and only the first cause is the reason.
83
+ let closedBy: "client disconnect" | "timeout" | null = null;
84
+
79
85
  // Client disconnect handler
80
86
  req.on("close", () => {
81
- console.log(`[HttpConnection:${connectionId}] CLIENT DISCONNECTED`);
82
87
  clientDisconnected = true;
83
88
  if (!streamClosed) {
84
- console.log(`[HttpConnection:${connectionId}] Closing stream due to client disconnect`);
85
- response.close();
89
+ closedBy = "client disconnect";
86
90
  streamClosed = true;
91
+ response.close();
87
92
  }
88
93
  });
89
-
94
+
90
95
  // Timeout handler
91
96
  const timeout = setTimeout(() => {
92
- console.log(`[HttpConnection:${connectionId}] TIMEOUT TRIGGERED (5 minutes)`);
93
- timeoutTriggered = true;
94
97
  if (!streamClosed) {
95
- console.log(`[HttpConnection:${connectionId}] Closing stream due to timeout`);
96
- response.close();
98
+ closedBy = "timeout";
97
99
  streamClosed = true;
100
+ response.close();
98
101
  }
99
102
  }, 5 * 60 * 1000);
100
-
103
+
101
104
  let messageCount = 0;
102
-
105
+
103
106
  response
104
107
  .next(data => {
105
108
  messageCount++;
106
109
  const writeStart = Date.now();
107
-
110
+
108
111
  if (clientDisconnected) {
109
112
  console.warn(`[HttpConnection:${connectionId}] Attempted to write to disconnected client - Message #${messageCount}`);
110
113
  return;
111
114
  }
112
-
115
+
113
116
  try {
114
117
  const jsonData = JSON.stringify(data) + "\n\n";
115
118
  res.write(jsonData);
116
119
  const writeDuration = Date.now() - writeStart;
117
-
118
- console.log(`[HttpConnection:${connectionId}] Message #${messageCount} sent - Size: ${jsonData.length} bytes, Duration: ${writeDuration}ms`);
119
-
120
+
120
121
  if (writeDuration > 50) {
121
122
  console.warn(`[HttpConnection:${connectionId}] SLOW write operation - Duration: ${writeDuration}ms`);
122
123
  }
@@ -125,24 +126,19 @@ function getOrStream<U>(
125
126
  }
126
127
  })
127
128
  .done(() => {
128
- console.log(`[HttpConnection:${connectionId}] STREAM DONE - Messages sent: ${messageCount}, Client disconnected: ${clientDisconnected}, Timeout triggered: ${timeoutTriggered}`);
129
-
130
129
  clearTimeout(timeout);
131
130
  streamClosed = true;
132
-
131
+
132
+ console.log(`[HttpConnection:${connectionId}] Stream ended - Messages sent: ${messageCount}, Duration: ${Date.now() - openedAt}ms, Reason: ${closedBy ?? "done"}`);
133
+
133
134
  try {
134
135
  if (!clientDisconnected) {
135
- console.log(`[HttpConnection:${connectionId}] Ending socket connection`);
136
136
  res.socket?.end();
137
- } else {
138
- console.log(`[HttpConnection:${connectionId}] Client already disconnected - skipping socket end`);
139
137
  }
140
138
  } catch (error) {
141
139
  console.error(`[HttpConnection:${connectionId}] ERROR ending socket: ${error}`);
142
140
  }
143
141
  });
144
-
145
- console.log(`[HttpConnection:${connectionId}] Stream handlers configured`);
146
142
  }
147
143
  })
148
144
  .catch(error => {
@@ -151,16 +147,13 @@ function getOrStream<U>(
151
147
  });
152
148
  }
153
149
  else {
154
- console.log(`[HttpConnection:${connectionId}] Regular GET request`);
155
150
  getMethod(user, req.params, req.query)
156
151
  .then(response => {
157
152
  if (!response) {
158
- console.log(`[HttpConnection:${connectionId}] Get method returned null - sending 404`);
159
- res.sendStatus(404);
153
+ sendRouteMiss(res);
160
154
  next();
161
155
  }
162
156
  else {
163
- console.log(`[HttpConnection:${connectionId}] Sending JSON response`);
164
157
  res.type("json");
165
158
  res.send(JSON.stringify(response));
166
159
  next();
@@ -199,9 +192,14 @@ function post<T, U>(
199
192
  ): Handler {
200
193
  return (req, res, next) => {
201
194
  const user = <RequestUser>(req as any).user;
202
- const message = parse(req.body);
195
+ const parsed = parseRequestInput(() => parse(req.body), req, res, next);
196
+ if (!parsed.ok) {
197
+ return;
198
+ }
199
+ const message = parsed.value;
203
200
  if (!message) {
204
- throw new Error('Ensure that you have called app.use(express.json()).');
201
+ handleError(new Invalid(`${MISSING_BODY} ${describeContentType(req)}`), req, res, next);
202
+ return;
205
203
  }
206
204
  method(user, message, req.params)
207
205
  .then(response => {
@@ -224,8 +222,11 @@ function postCreate<T>(
224
222
  ): Handler {
225
223
  return (req, res, next) => {
226
224
  const user = <RequestUser>(req as any).user;
227
- const message = parse(req);
228
- method(user, message)
225
+ const parsed = parseRequestInput(() => parse(req), req, res, next);
226
+ if (!parsed.ok) {
227
+ return;
228
+ }
229
+ method(user, parsed.value)
229
230
  .then(_ => {
230
231
  res.sendStatus(201);
231
232
  next();
@@ -237,19 +238,16 @@ function postCreate<T>(
237
238
  function postStringCreate(method: (user: RequestUser, message: string) => Promise<void>): Handler {
238
239
  return (req, res, next) => {
239
240
  const user = <RequestUser>(req as any).user;
240
- const input = parseString(req.body);
241
- if (!input || typeof (input) !== 'string') {
242
- res.type("text");
243
- res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
244
- }
245
- else {
246
- method(user, input)
247
- .then(_ => {
248
- res.sendStatus(201);
249
- next();
250
- })
251
- .catch(error => handleError(error, req, res, next));
241
+ const parsed = parseRequestInput(() => parseTextBody(req), req, res, next);
242
+ if (!parsed.ok) {
243
+ return;
252
244
  }
245
+ method(user, parsed.value)
246
+ .then(_ => {
247
+ res.sendStatus(201);
248
+ next();
249
+ })
250
+ .catch(error => handleError(error, req, res, next));
253
251
  };
254
252
  }
255
253
 
@@ -261,29 +259,27 @@ function postReadWithStreaming(
261
259
  ): Handler {
262
260
  return (req, res, next) => {
263
261
  const user = <RequestUser>(req as any).user;
264
- const input = parseString(req.body);
265
- if (!input || typeof (input) !== 'string') {
266
- res.type("text");
267
- res.status(500).send('Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).');
262
+ const parsed = parseRequestInput(() => parseTextBody(req), req, res, next);
263
+ if (!parsed.ok) {
264
+ return;
268
265
  }
269
- else {
270
- // Check if Accept header explicitly prefers a specific format
271
- const acceptHeader = req.get('Accept');
272
- let acceptType: string = 'text/plain'; // Default for backward compatibility
273
-
274
- if (acceptHeader && acceptHeader !== '*/*') {
275
- // Only use req.accepts() when there's a specific preference
276
- const preferredType = req.accepts(['text/csv', 'application/x-ndjson', 'application/json', 'text/plain']);
277
- acceptType = preferredType ? String(preferredType) : 'text/plain';
278
- }
279
-
280
- method(user, input, acceptType)
281
- .then(({resultStream, csvMetadata}) => {
282
- outputReadResults(resultStream, res, acceptType, csvMetadata);
283
- next();
284
- })
285
- .catch(error => handleError(error, req, res, next));
266
+
267
+ // Check if Accept header explicitly prefers a specific format
268
+ const acceptHeader = req.get('Accept');
269
+ let acceptType: string = 'text/plain'; // Default for backward compatibility
270
+
271
+ if (acceptHeader && acceptHeader !== '*/*') {
272
+ // Only use req.accepts() when there's a specific preference
273
+ const preferredType = req.accepts(['text/csv', 'application/x-ndjson', 'application/json', 'text/plain']);
274
+ acceptType = preferredType ? String(preferredType) : 'text/plain';
286
275
  }
276
+
277
+ method(user, parsed.value, acceptType)
278
+ .then(({resultStream, csvMetadata}) => {
279
+ outputReadResults(resultStream, res, acceptType, csvMetadata);
280
+ next();
281
+ })
282
+ .catch(error => handleError(error, req, res, next));
287
283
  };
288
284
  }
289
285
 
@@ -313,7 +309,7 @@ function inputSaveMessage(req: Request): GraphSource {
313
309
  read: async (onEnvelopes) => {
314
310
  const message = parseSaveMessage(req.body);
315
311
  if (!message) {
316
- throw new Error('Ensure that you have called app.use(express.json()).');
312
+ throw new Invalid(MISSING_JSON_BODY);
317
313
  }
318
314
  await onEnvelopes(message.facts.map(fact => ({
319
315
  fact: fact,
@@ -358,7 +354,9 @@ function outputReadResults(
358
354
  .catch(error => {
359
355
  console.error('Error in outputReadResults:', error);
360
356
  if (!res.headersSent) {
361
- res.status(500).send('Internal server error');
357
+ // Without an explicit type, res.send of a string makes Express
358
+ // infer text/html and label an error body as markup.
359
+ res.type("text").status(500).send('Internal server error');
362
360
  }
363
361
  });
364
362
  }
@@ -550,11 +548,19 @@ export class HttpRouter {
550
548
  }
551
549
 
552
550
  const userIdentity = serializeUserIdentity(user);
553
- await this.authorization.save(userIdentity, factRecords
551
+ const envelopes: FactEnvelope[] = factRecords
554
552
  .map(fact => ({
555
553
  fact: fact,
556
554
  signatures: []
557
- })));
555
+ }));
556
+ try {
557
+ await this.authorization.save(userIdentity, envelopes);
558
+ } catch (error) {
559
+ // /write reaches the same authorization path as /save, so it
560
+ // needs the same translation of predecessor-resolution
561
+ // failures into a diagnosable 4xx (issue #182 finding 2).
562
+ throw toSaveAuthorizationError(error, envelopes);
563
+ }
558
564
  });
559
565
  }
560
566
 
@@ -988,34 +994,174 @@ export class FeedNotFound extends Error {
988
994
  }
989
995
  }
990
996
 
997
+ // Both message shapes below are matched by prose, because jinaga reports a
998
+ // missing predecessor as a plain Error. That is fragile: an upstream wording
999
+ // change silently turns these back into 500s (issue #182 finding 5). The
1000
+ // end-to-end cases in test/http/errorStatusMappingSpec.ts drive the real
1001
+ // AuthorizationEngine so that drift fails the build.
1002
+ //
1003
+ // jinaga 6.11.3 landed the typed errors of jinaga.js#234, so the topological
1004
+ // sort's failure is now an instanceof check below. One prose pattern remains:
1005
+ // specification-runner still reports its missing fact as a plain Error.
1006
+
991
1007
  // Authorization rule evaluation reports a missing predecessor as a plain
992
- // Error of this shape (see authorizationRules.ts / specification-runner.ts
993
- // in jinaga). It surfaces when the client's save request didn't include the
994
- // full closure of facts a rule needs to walk. Map it to a diagnosable 4xx
995
- // instead of an opaque 500 (issue #175).
1008
+ // Error of this shape (specification-runner.ts in jinaga, still untyped as of
1009
+ // 6.11.3). It surfaces when the client's save request didn't include the full
1010
+ // closure of facts a rule needs to walk. Map it to a diagnosable 4xx instead
1011
+ // of an opaque 500 (issue #175).
996
1012
  const FACT_NOT_DEFINED_PATTERN = /^The fact (\S+):(\S+) is not defined\.$/;
997
1013
 
998
1014
  function toSaveAuthorizationError(error: any, envelopes: FactEnvelope[]): any {
999
- if (error instanceof Error) {
1000
- const match = FACT_NOT_DEFINED_PATTERN.exec(error.message);
1001
- if (match) {
1002
- const [, factType, factHash] = match;
1003
- const factTypesInBatch = Array.from(new Set(envelopes.map(e => e.fact.type))).join(", ");
1004
- Trace.warn(
1005
- `Save authorization could not resolve predecessor ${factType}:${factHash} ` +
1006
- `among ${envelopes.length} fact(s) in the request (types: ${factTypesInBatch}).`
1015
+ if (!(error instanceof Error)) {
1016
+ return error;
1017
+ }
1018
+
1019
+ const describeBatch = () => {
1020
+ const factTypesInBatch = Array.from(new Set(envelopes.map(e => e.fact.type))).join(", ");
1021
+ return `${envelopes.length} fact(s) in the request (types: ${factTypesInBatch})`;
1022
+ };
1023
+
1024
+ // The batch named a predecessor that is in neither the batch nor storage,
1025
+ // so the topological sort could never reach the facts that depend on it.
1026
+ // A client data problem, not a server fault (issue #182 finding 1).
1027
+ if (error instanceof PredecessorNotResolvedError) {
1028
+ const missing = error.missingPredecessors.map(r => `${r.type}:${r.hash}`);
1029
+ const unresolved = error.unresolvedFacts.map(r => `${r.type}:${r.hash}`);
1030
+ Trace.warn(
1031
+ `Save authorization could not resolve ${missing.length} predecessor(s) [${missing.join(", ")}] ` +
1032
+ `needed by ${unresolved.length} fact(s) [${unresolved.join(", ")}] among ${describeBatch()}.`
1033
+ );
1034
+ const advice = `Send the full closure of predecessors together, or commit the predecessors first.`;
1035
+ if (missing.length === 1) {
1036
+ return new Invalid(
1037
+ `The fact ${missing[0]} is named as a predecessor, but it was not found in storage ` +
1038
+ `and was not included in the request. ${advice}`
1007
1039
  );
1040
+ }
1041
+ if (missing.length > 1) {
1008
1042
  return new Invalid(
1009
- `The fact ${factType}:${factHash} is required to authorize this save but was not included in the request. ` +
1010
- `Ensure the full closure of predecessors needed by your authorization rules is sent together.`
1043
+ `${missing.length} facts are named as predecessors, but they were not found in storage ` +
1044
+ `and were not included in the request: ${missing.join(", ")}. ${advice}`
1011
1045
  );
1012
1046
  }
1047
+ // The engine could not attribute the failure to specific references.
1048
+ return new Invalid(
1049
+ `${unresolved.length} fact(s) could not be authorized because one or more predecessors ` +
1050
+ `were not found in storage and were not included in the request. ${advice}`
1051
+ );
1052
+ }
1053
+
1054
+ const notDefined = FACT_NOT_DEFINED_PATTERN.exec(error.message);
1055
+ if (notDefined) {
1056
+ const [, factType, factHash] = notDefined;
1057
+ Trace.warn(
1058
+ `Save authorization could not resolve predecessor ${factType}:${factHash} ` +
1059
+ `among ${describeBatch()}.`
1060
+ );
1061
+ return new Invalid(
1062
+ `The fact ${factType}:${factHash} is required to authorize this save but was not included in the request. ` +
1063
+ `Ensure the full closure of predecessors needed by your authorization rules is sent together.`
1064
+ );
1013
1065
  }
1066
+
1014
1067
  return error;
1015
1068
  }
1016
1069
 
1070
+ // post() serves both a JSON route (/load) and a text route (/feeds), so its
1071
+ // message names no particular parser; the single-content-type entry points
1072
+ // keep the specific hint they have always given.
1073
+ const MISSING_BODY = 'The request body is empty or could not be read. Ensure that the body parser for this Content-Type is registered.';
1074
+ const MISSING_JSON_BODY = 'Ensure that you have called app.use(express.json()).';
1075
+ const MISSING_TEXT_BODY = 'Expected Content-Type text/plain. Ensure that you have called app.use(express.text()).';
1076
+
1077
+ // A request body that fails to parse is a client error. Both this module's
1078
+ // parseString and jinaga's message parsers signal that by throwing
1079
+ // synchronously from the handler, where Express hands it to its default error
1080
+ // handler and reports 500. Classify parse failures as Invalid so handleError
1081
+ // renders a 400 (issue #182 finding 3).
1082
+ function toInvalidInput(error: any): Invalid {
1083
+ if (error instanceof Invalid) {
1084
+ return error;
1085
+ }
1086
+ return new Invalid(error instanceof Error ? error.message : String(error));
1087
+ }
1088
+
1089
+ // /read and /write accept text/plain only, so every unreadable body there has
1090
+ // the same cause and deserves the same actionable hint. Checking req.body here
1091
+ // rather than deferring to parseString keeps a missing body parser — the common
1092
+ // misconfiguration — from being reported with parseString's generic
1093
+ // content-type message.
1094
+ function parseTextBody(req: Request): string {
1095
+ const body = req.body;
1096
+ if (typeof body !== 'string' || !body) {
1097
+ throw new Invalid(`${MISSING_TEXT_BODY} ${describeContentType(req)}`);
1098
+ }
1099
+ return body;
1100
+ }
1101
+
1102
+ // Naming what arrived turns "check your setup" into something the caller can
1103
+ // act on without guessing which end is misconfigured. The value is echoed from
1104
+ // the request, so cap it: a real content type is short, and the body should
1105
+ // not grow with whatever the caller chose to send.
1106
+ const MAX_ECHOED_CONTENT_TYPE = 80;
1107
+
1108
+ function describeContentType(req: Request): string {
1109
+ const contentType = req.get('Content-Type');
1110
+ if (!contentType) {
1111
+ return `No Content-Type header was received.`;
1112
+ }
1113
+ const echoed = contentType.length > MAX_ECHOED_CONTENT_TYPE
1114
+ ? `${contentType.substring(0, MAX_ECHOED_CONTENT_TYPE)}…`
1115
+ : contentType;
1116
+ return `Received Content-Type: ${echoed}.`;
1117
+ }
1118
+
1119
+ function parseRequestInput<T>(
1120
+ parse: () => T,
1121
+ req: Request,
1122
+ res: Response,
1123
+ next: NextFunction
1124
+ ): { ok: true, value: T } | { ok: false } {
1125
+ try {
1126
+ return { ok: true, value: parse() };
1127
+ } catch (error) {
1128
+ handleError(toInvalidInput(error), req, res, next);
1129
+ return { ok: false };
1130
+ }
1131
+ }
1132
+
1133
+ // A route miss (no :hash in the URL) is a 404 just like a feed-cache miss,
1134
+ // but for a different reason. Send a machine-readable body so a client can
1135
+ // always tell "wrong URL" from "re-subscribe via POST /feeds" (issue #182
1136
+ // finding 6); handleError sends "feed_not_found" for the latter.
1137
+ function sendRouteMiss(res: Response) {
1138
+ res.type("text");
1139
+ res.status(404).send("not_found");
1140
+ }
1141
+
1142
+ // These bodies are diagnostics, not markup, but they quote client-supplied
1143
+ // text: a Content-Type header, a fact type named in a parse failure. The
1144
+ // response is text/plain and marked nosniff, yet that is the content type
1145
+ // saying so — neutralize the characters that could start markup here, at the
1146
+ // one place every error body is written, so the guarantee does not rest on a
1147
+ // header a proxy or a later edit could change. Quotes are left alone: they
1148
+ // only matter inside a tag, which escaping '<' already prevents, and messages
1149
+ // quote identifiers routinely.
1150
+ const HTML_META_CHARACTERS: { [character: string]: string } = {
1151
+ "&": "&amp;",
1152
+ "<": "&lt;",
1153
+ ">": "&gt;"
1154
+ };
1155
+
1156
+ function sanitizeErrorBody(message: string): string {
1157
+ return message.replace(/[&<>]/g, character => HTML_META_CHARACTERS[character]);
1158
+ }
1159
+
1017
1160
  function handleError(error: any, req: Request, res: Response, next: NextFunction) {
1018
1161
  const requestPath = req.path;
1162
+ // Error bodies are plain text built partly from client-supplied input, so
1163
+ // stop a browser from sniffing one as markup.
1164
+ res.set("X-Content-Type-Options", "nosniff");
1019
1165
  if (error instanceof FeedNotFound) {
1020
1166
  Trace.warn(`Feed not found: ${error.feedHash} (Path: ${requestPath})`);
1021
1167
  res.type("text");
@@ -1023,15 +1169,19 @@ function handleError(error: any, req: Request, res: Response, next: NextFunction
1023
1169
  } else if (error instanceof Forbidden) {
1024
1170
  Trace.warn(`Forbidden: ${error.message} (Path: ${requestPath})`);
1025
1171
  res.type("text");
1026
- res.status(403).send(error.message);
1172
+ res.status(403).send(sanitizeErrorBody(error.message));
1027
1173
  } else if (error instanceof Invalid) {
1028
1174
  Trace.warn(`Invalid: ${error.message} (Path: ${requestPath})`);
1029
1175
  res.type("text");
1030
- res.status(400).send(error.message);
1176
+ res.status(400).send(sanitizeErrorBody(error.message));
1031
1177
  } else {
1178
+ // Classification above is an allow-list, so anything reaching here is
1179
+ // unclassified — including genuine internal failures whose message can
1180
+ // name internal hosts or credentials. Keep the detail in the trace and
1181
+ // send a generic body to the caller (issue #182 finding 4).
1032
1182
  Trace.error(`Error: ${error.message} (Path: ${requestPath})`);
1033
1183
  res.type("text");
1034
- res.status(500).send(error.message);
1184
+ res.status(500).send("Internal server error");
1035
1185
  }
1036
1186
  next();
1037
1187
  }
@@ -1,14 +1,18 @@
1
+ // This class logs nothing on its success path. A stream's lifetime is reported
2
+ // once by the request that opened it (see getOrStream in router.ts), and a line
3
+ // per feed, per replayed item, or per done handler says the same thing again
4
+ // once per unit of traffic. Issue #192: those per-item lines were 11.9% of a
5
+ // 2.8-million-line production log. Failures still log, because a failed handler
6
+ // is an event an operator cannot reconstruct from the summary.
1
7
  export class Stream<T> {
2
8
  private queue: T[] = [];
3
9
  private handlers: ((data: T) => void)[] = [];
4
10
  private doneHandlers: (() => void)[] = [];
5
11
  private closed = false;
6
12
  private readonly streamId: string;
7
- private feedCount = 0;
8
13
 
9
14
  constructor() {
10
15
  this.streamId = Math.random().toString(36).substring(2, 10);
11
- console.log(`[Stream:${this.streamId}] Created new stream`);
12
16
  }
13
17
 
14
18
  next(handler: (data: T) => void): Stream<T> {
@@ -16,21 +20,18 @@ export class Stream<T> {
16
20
  console.warn(`[Stream:${this.streamId}] Attempted to add handler to closed stream`);
17
21
  return this;
18
22
  }
19
-
23
+
20
24
  this.handlers.push(handler);
21
- const queueLength = this.queue.length;
22
- console.log(`[Stream:${this.streamId}] Added handler - Total handlers: ${this.handlers.length}, Queued items: ${queueLength}`);
23
-
25
+
24
26
  // Replay queued data to new handler
25
27
  this.queue.forEach((data, index) => {
26
28
  try {
27
29
  handler(data);
28
- console.log(`[Stream:${this.streamId}] Replayed queued item ${index + 1}/${queueLength} to new handler`);
29
30
  } catch (error) {
30
31
  console.error(`[Stream:${this.streamId}] Error replaying queued item ${index + 1}: ${error}`);
31
32
  }
32
33
  });
33
-
34
+
34
35
  return this;
35
36
  }
36
37
 
@@ -44,9 +45,8 @@ export class Stream<T> {
44
45
  }
45
46
  return this;
46
47
  }
47
-
48
+
48
49
  this.doneHandlers.push(handler);
49
- console.log(`[Stream:${this.streamId}] Added done handler - Total done handlers: ${this.doneHandlers.length}`);
50
50
  return this;
51
51
  }
52
52
 
@@ -55,70 +55,55 @@ export class Stream<T> {
55
55
  console.warn(`[Stream:${this.streamId}] Attempted to close already closed stream`);
56
56
  return;
57
57
  }
58
-
59
- console.log(`[Stream:${this.streamId}] CLOSING stream - Handlers: ${this.handlers.length}, Done handlers: ${this.doneHandlers.length}, Queue: ${this.queue.length}, Feed count: ${this.feedCount}`);
60
-
58
+
61
59
  this.closed = true;
62
-
60
+
63
61
  // Execute done handlers
64
- const doneHandlerCount = this.doneHandlers.length;
65
62
  this.doneHandlers.forEach((handler, index) => {
66
63
  try {
67
64
  handler();
68
- console.log(`[Stream:${this.streamId}] Executed done handler ${index + 1}/${doneHandlerCount}`);
69
65
  } catch (error) {
70
66
  console.error(`[Stream:${this.streamId}] Error in done handler ${index + 1}: ${error}`);
71
67
  }
72
68
  });
73
-
69
+
74
70
  // Clear all references
75
71
  this.queue = [];
76
72
  this.handlers = [];
77
73
  this.doneHandlers = [];
78
-
79
- console.log(`[Stream:${this.streamId}] Stream closed and cleaned up`);
80
74
  }
81
75
 
82
76
  feed(data: any): void {
83
77
  const feedStart = Date.now();
84
-
78
+
85
79
  if (this.closed) {
86
80
  console.warn(`[Stream:${this.streamId}] Attempted to feed data to closed stream - Data type: ${typeof data}`);
87
81
  return;
88
82
  }
89
-
90
- this.feedCount++;
83
+
91
84
  const handlerCount = this.handlers.length;
92
-
93
- console.log(`[Stream:${this.streamId}] FEEDING data - Feed #${this.feedCount}, Handlers: ${handlerCount}, Queue size before: ${this.queue.length}`);
94
-
85
+
95
86
  this.queue.push(data);
96
-
87
+
97
88
  // Notify all handlers
98
- let successCount = 0;
99
- let errorCount = 0;
100
-
101
89
  this.handlers.forEach((handler, index) => {
102
90
  try {
103
91
  const handlerStart = Date.now();
104
92
  handler(data);
105
93
  const handlerDuration = Date.now() - handlerStart;
106
- successCount++;
107
-
94
+
108
95
  if (handlerDuration > 50) {
109
96
  console.warn(`[Stream:${this.streamId}] SLOW handler - Handler ${index + 1}/${handlerCount}, Duration: ${handlerDuration}ms`);
110
97
  }
111
98
  } catch (error) {
112
- errorCount++;
113
99
  console.error(`[Stream:${this.streamId}] ERROR in handler ${index + 1}/${handlerCount}: ${error}`);
114
100
  }
115
101
  });
116
-
102
+
117
103
  const totalDuration = Date.now() - feedStart;
118
- console.log(`[Stream:${this.streamId}] Feed complete - Success: ${successCount}, Errors: ${errorCount}, Queue size after: ${this.queue.length}, Duration: ${totalDuration}ms`);
119
-
104
+
120
105
  if (totalDuration > 100) {
121
106
  console.warn(`[Stream:${this.streamId}] SLOW feed operation - Duration: ${totalDuration}ms`);
122
107
  }
123
108
  }
124
- }
109
+ }