@hasna/logs 0.3.26 → 0.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -10
- package/dashboard/dist/assets/index-C0wZYq1m.js +53 -0
- package/dashboard/dist/assets/index-DGNrK5qb.css +1 -0
- package/dashboard/dist/index.html +14 -0
- package/dist/cli/index.js +8511 -177
- package/dist/count-bmj4r2zb.js +10 -0
- package/dist/{diagnose-e0w5rwbc.js → diagnose-3q5cy9ra.js} +2 -2
- package/dist/{export-c3eqjste.js → export-cngdb9fh.js} +1 -1
- package/dist/{http-zm3ph78w.js → http-r0xc3d2s.js} +79 -8
- package/dist/index-931pbyn5.js +141 -0
- package/dist/index-b5c72f1p.js +7 -0
- package/dist/{index-gc0zvs88.js → index-bnr19y0h.js} +596 -37
- package/dist/{index-7w7v7hnr.js → index-by1pdzbr.js} +14 -5
- package/dist/{index-3dr7d80h.js → index-e1930v9b.js} +12 -8
- package/dist/{index-eh9bkbpa.js → index-e72k53yq.js} +10 -2
- package/dist/{index-edn08m6f.js → index-gcd14q2f.js} +9 -6
- package/dist/index-hq6kzaah.js +26 -0
- package/dist/index-j34f36wy.js +5672 -0
- package/dist/index-p4dbdzx4.js +1849 -0
- package/dist/{index-5qznfyah.js → index-q27bgpr1.js} +1086 -1646
- package/dist/index-t3x838zw.js +2583 -0
- package/dist/{index-ww5ggfv3.js → index-zkb3z95a.js} +12 -9
- package/dist/index.js +2982 -22
- package/dist/{jobs-ypmmc2ma.js → jobs-hsgyhfvm.js} +2 -1
- package/dist/mcp/index.js +1473 -4286
- package/dist/{query-7jwj05er.js → query-c5a43zx3.js} +3 -2
- package/dist/server/index.js +2944 -417
- package/dist/storage.js +50 -0
- package/package.json +27 -8
- package/biome.json +0 -13
- package/bun.lock +0 -376
- package/dashboard/README.md +0 -73
- package/dashboard/bun.lock +0 -526
- package/dashboard/eslint.config.js +0 -23
- package/dashboard/index.html +0 -13
- package/dashboard/package.json +0 -32
- package/dashboard/src/App.css +0 -184
- package/dashboard/src/App.tsx +0 -49
- package/dashboard/src/api.ts +0 -33
- package/dashboard/src/assets/hero.png +0 -0
- package/dashboard/src/assets/react.svg +0 -1
- package/dashboard/src/assets/vite.svg +0 -1
- package/dashboard/src/index.css +0 -111
- package/dashboard/src/main.tsx +0 -10
- package/dashboard/src/pages/Alerts.tsx +0 -69
- package/dashboard/src/pages/Issues.tsx +0 -50
- package/dashboard/src/pages/Perf.tsx +0 -75
- package/dashboard/src/pages/Projects.tsx +0 -67
- package/dashboard/src/pages/Summary.tsx +0 -67
- package/dashboard/src/pages/Tail.tsx +0 -65
- package/dashboard/tsconfig.app.json +0 -28
- package/dashboard/tsconfig.json +0 -7
- package/dashboard/tsconfig.node.json +0 -26
- package/dashboard/vite.config.ts +0 -14
- package/dist/count-x3n7qg3c.js +0 -9
- package/dist/index-997bkzr2.js +0 -15
- package/dist/index-pen6t0yc.js +0 -10794
- package/sdk/package.json +0 -27
- package/sdk/src/index.ts +0 -143
- package/sdk/src/types.ts +0 -56
- package/src/cli/entrypoints.test.ts +0 -63
- package/src/cli/index.ts +0 -471
- package/src/db/index.test.ts +0 -33
- package/src/db/index.ts +0 -189
- package/src/db/migrations/001_alert_rules.ts +0 -21
- package/src/db/migrations/002_issues.ts +0 -21
- package/src/db/migrations/003_retention.ts +0 -15
- package/src/db/migrations/004_page_auth.ts +0 -13
- package/src/db/pg-migrations.ts +0 -167
- package/src/index.ts +0 -1
- package/src/lib/alerts.test.ts +0 -67
- package/src/lib/alerts.ts +0 -117
- package/src/lib/browser-script.test.ts +0 -35
- package/src/lib/browser-script.ts +0 -31
- package/src/lib/compare.test.ts +0 -52
- package/src/lib/compare.ts +0 -85
- package/src/lib/count.test.ts +0 -44
- package/src/lib/count.ts +0 -55
- package/src/lib/diagnose.test.ts +0 -55
- package/src/lib/diagnose.ts +0 -91
- package/src/lib/export.test.ts +0 -66
- package/src/lib/export.ts +0 -65
- package/src/lib/github.ts +0 -38
- package/src/lib/health.test.ts +0 -48
- package/src/lib/health.ts +0 -51
- package/src/lib/ingest.test.ts +0 -57
- package/src/lib/ingest.ts +0 -78
- package/src/lib/issues.test.ts +0 -79
- package/src/lib/issues.ts +0 -70
- package/src/lib/jobs.test.ts +0 -69
- package/src/lib/jobs.ts +0 -63
- package/src/lib/lighthouse.ts +0 -65
- package/src/lib/package-meta.test.ts +0 -43
- package/src/lib/package-meta.ts +0 -80
- package/src/lib/page-auth.test.ts +0 -54
- package/src/lib/page-auth.ts +0 -48
- package/src/lib/parse-time.test.ts +0 -37
- package/src/lib/parse-time.ts +0 -14
- package/src/lib/perf.test.ts +0 -45
- package/src/lib/perf.ts +0 -46
- package/src/lib/projects.test.ts +0 -73
- package/src/lib/projects.ts +0 -69
- package/src/lib/query.test.ts +0 -104
- package/src/lib/query.ts +0 -84
- package/src/lib/retention.test.ts +0 -42
- package/src/lib/retention.ts +0 -62
- package/src/lib/rotate.test.ts +0 -37
- package/src/lib/rotate.ts +0 -27
- package/src/lib/scanner.ts +0 -131
- package/src/lib/scheduler.ts +0 -63
- package/src/lib/session-context.ts +0 -28
- package/src/lib/summarize.test.ts +0 -38
- package/src/lib/summarize.ts +0 -23
- package/src/mcp/http.test.ts +0 -92
- package/src/mcp/http.ts +0 -135
- package/src/mcp/index.test.ts +0 -27
- package/src/mcp/index.ts +0 -444
- package/src/server/index.ts +0 -61
- package/src/server/routes/alerts.ts +0 -32
- package/src/server/routes/issues.ts +0 -43
- package/src/server/routes/jobs.ts +0 -32
- package/src/server/routes/logs.ts +0 -113
- package/src/server/routes/perf.ts +0 -23
- package/src/server/routes/projects.ts +0 -67
- package/src/server/routes/stream.ts +0 -43
- package/src/server/server.test.ts +0 -194
- package/src/types/index.ts +0 -119
- package/tsconfig.json +0 -22
- /package/dashboard/{public → dist}/favicon.svg +0 -0
- /package/dashboard/{public → dist}/icons.svg +0 -0
|
@@ -0,0 +1,2583 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
sqlBindings
|
|
4
|
+
} from "./index-b5c72f1p.js";
|
|
5
|
+
|
|
6
|
+
// src/lib/issues.ts
|
|
7
|
+
import { createHash } from "crypto";
|
|
8
|
+
function computeFingerprint(level, service, message, stackTrace) {
|
|
9
|
+
const normalized = message.replace(/[0-9a-f]{8,}/gi, "<id>").replace(/\d+/g, "<n>").replace(/https?:\/\/[^\s]+/g, "<url>").trim();
|
|
10
|
+
const stackFrame = stackTrace ? stackTrace.split(`
|
|
11
|
+
`).slice(0, 3).join("|") : "";
|
|
12
|
+
const raw = `${level}|${service ?? ""}|${normalized}|${stackFrame}`;
|
|
13
|
+
return createHash("sha256").update(raw).digest("hex").slice(0, 16);
|
|
14
|
+
}
|
|
15
|
+
function upsertIssue(db, data) {
|
|
16
|
+
const fingerprint = computeFingerprint(data.level, data.service ?? null, data.message, data.stack_trace);
|
|
17
|
+
return db.prepare(`
|
|
18
|
+
INSERT INTO issues (project_id, fingerprint, level, service, message_template)
|
|
19
|
+
VALUES ($project_id, $fingerprint, $level, $service, $message_template)
|
|
20
|
+
ON CONFLICT(project_id, fingerprint) DO UPDATE SET
|
|
21
|
+
count = count + 1,
|
|
22
|
+
last_seen = strftime('%Y-%m-%dT%H:%M:%fZ','now'),
|
|
23
|
+
status = CASE WHEN status = 'resolved' THEN 'open' ELSE status END
|
|
24
|
+
RETURNING *
|
|
25
|
+
`).get({
|
|
26
|
+
$project_id: data.project_id ?? null,
|
|
27
|
+
$fingerprint: fingerprint,
|
|
28
|
+
$level: data.level,
|
|
29
|
+
$service: data.service ?? null,
|
|
30
|
+
$message_template: data.message.slice(0, 500)
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
function listIssues(db, projectId, status, limit = 50) {
|
|
34
|
+
const conditions = [];
|
|
35
|
+
const params = { $limit: limit };
|
|
36
|
+
if (projectId) {
|
|
37
|
+
conditions.push("project_id = $p");
|
|
38
|
+
params.$p = projectId;
|
|
39
|
+
}
|
|
40
|
+
if (status) {
|
|
41
|
+
conditions.push("status = $status");
|
|
42
|
+
params.$status = status;
|
|
43
|
+
}
|
|
44
|
+
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
45
|
+
return db.prepare(`SELECT * FROM issues ${where} ORDER BY last_seen DESC LIMIT $limit`).all(sqlBindings(params));
|
|
46
|
+
}
|
|
47
|
+
function getIssue(db, id) {
|
|
48
|
+
return db.prepare("SELECT * FROM issues WHERE id = $id").get(sqlBindings({ $id: id }));
|
|
49
|
+
}
|
|
50
|
+
function updateIssueStatus(db, id, status) {
|
|
51
|
+
return db.prepare("UPDATE issues SET status = $status WHERE id = $id RETURNING *").get(sqlBindings({ $id: id, $status: status }));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// src/lib/event-store.ts
|
|
55
|
+
import { createHash as createHash4 } from "crypto";
|
|
56
|
+
import {
|
|
57
|
+
closeSync,
|
|
58
|
+
existsSync,
|
|
59
|
+
fsyncSync,
|
|
60
|
+
mkdirSync,
|
|
61
|
+
openSync,
|
|
62
|
+
readFileSync,
|
|
63
|
+
readdirSync,
|
|
64
|
+
renameSync,
|
|
65
|
+
rmSync,
|
|
66
|
+
statSync,
|
|
67
|
+
writeFileSync,
|
|
68
|
+
writeSync
|
|
69
|
+
} from "fs";
|
|
70
|
+
import { dirname, join, resolve, sep } from "path";
|
|
71
|
+
|
|
72
|
+
// src/lib/source-map-projections.ts
|
|
73
|
+
import { createHash as createHash2 } from "crypto";
|
|
74
|
+
var MAX_PROJECTED_SOURCE_MAP_SOURCES = 200;
|
|
75
|
+
var MAX_SOURCE_MAP_STRING = 500;
|
|
76
|
+
var MAX_SOURCE_MAP_ERROR = 240;
|
|
77
|
+
var SOURCE_MAP_ROOT_KEYS = [
|
|
78
|
+
"source_map_id",
|
|
79
|
+
"source_map_artifact_id",
|
|
80
|
+
"source_map_path",
|
|
81
|
+
"javascript_artifact_id",
|
|
82
|
+
"javascript_path",
|
|
83
|
+
"linked_by",
|
|
84
|
+
"file",
|
|
85
|
+
"sourceRoot",
|
|
86
|
+
"source_root",
|
|
87
|
+
"version",
|
|
88
|
+
"validation_status",
|
|
89
|
+
"validation_error",
|
|
90
|
+
"source_count",
|
|
91
|
+
"section_count",
|
|
92
|
+
"names_count",
|
|
93
|
+
"mappings_length",
|
|
94
|
+
"has_sources_content",
|
|
95
|
+
"sources",
|
|
96
|
+
"sections",
|
|
97
|
+
"sourcesContent",
|
|
98
|
+
"names",
|
|
99
|
+
"mappings",
|
|
100
|
+
"raw_json",
|
|
101
|
+
"source_storage_policy",
|
|
102
|
+
"projected_source_limit"
|
|
103
|
+
];
|
|
104
|
+
var SAFE_VALIDATION_ERRORS = new Set([
|
|
105
|
+
"source map exceeds bounded parser size",
|
|
106
|
+
"source map could not be read",
|
|
107
|
+
"source map JSON is invalid",
|
|
108
|
+
"source map root must be an object",
|
|
109
|
+
"source map version must be 3",
|
|
110
|
+
"source map sources must be an array",
|
|
111
|
+
"source map mappings must be a string"
|
|
112
|
+
]);
|
|
113
|
+
function sanitizeSourceMapTelemetry(value) {
|
|
114
|
+
const sourceMap = objectRecord(value);
|
|
115
|
+
if (Object.keys(sourceMap).length === 0)
|
|
116
|
+
return null;
|
|
117
|
+
if (!hasSourceMapTelemetrySignal(sourceMap))
|
|
118
|
+
return null;
|
|
119
|
+
const sources = sanitizedSources(sourceMap);
|
|
120
|
+
const rawSources = Array.isArray(sourceMap.sources) ? sourceMap.sources : [];
|
|
121
|
+
const rawSourcesContent = Array.isArray(sourceMap.sourcesContent) ? sourceMap.sourcesContent : [];
|
|
122
|
+
const rawNames = Array.isArray(sourceMap.names) ? sourceMap.names : [];
|
|
123
|
+
const rawMappings = typeof sourceMap.mappings === "string" ? sourceMap.mappings : null;
|
|
124
|
+
const sourceCount = integerValue(sourceMap.source_count) ?? rawSources.length;
|
|
125
|
+
const hasSourcesContent = booleanValue(sourceMap.has_sources_content) ?? (rawSourcesContent.some((content) => typeof content === "string") || sources.rows.some((source) => source.has_content));
|
|
126
|
+
const validation = validationStatus(sourceMap.validation_status) ?? (integerValue(sourceMap.version) === 3 && Array.isArray(sourceMap.sources) && typeof sourceMap.mappings === "string" ? "parsed" : null);
|
|
127
|
+
const sanitized = compactObject({
|
|
128
|
+
source_map_id: sanitizeSourceMapIdentifierValue(sourceMap.source_map_id),
|
|
129
|
+
source_map_artifact_id: sanitizeSourceMapIdentifierValue(sourceMap.source_map_artifact_id),
|
|
130
|
+
source_map_path: sanitizeSourceMapPathValue(sourceMap.source_map_path),
|
|
131
|
+
javascript_artifact_id: sanitizeSourceMapIdentifierValue(sourceMap.javascript_artifact_id),
|
|
132
|
+
javascript_path: sanitizeSourceMapPathValue(sourceMap.javascript_path),
|
|
133
|
+
linked_by: linkedBy(sourceMap.linked_by),
|
|
134
|
+
file: sanitizeSourceMapPathValue(sourceMap.file),
|
|
135
|
+
source_root: sanitizeSourceMapPathValue(sourceMap.source_root ?? sourceMap.sourceRoot),
|
|
136
|
+
version: integerValue(sourceMap.version),
|
|
137
|
+
validation_status: validation,
|
|
138
|
+
validation_error: sanitizeSourceMapValidationError(sourceMap.validation_error),
|
|
139
|
+
source_count: sourceCount,
|
|
140
|
+
section_count: Array.isArray(sourceMap.sections) ? sourceMap.sections.length : integerValue(sourceMap.section_count),
|
|
141
|
+
names_count: integerValue(sourceMap.names_count) ?? rawNames.length,
|
|
142
|
+
mappings_length: integerValue(sourceMap.mappings_length) ?? rawMappings?.length,
|
|
143
|
+
has_sources_content: hasSourcesContent,
|
|
144
|
+
sources: sources.rows,
|
|
145
|
+
truncated: booleanValue(sourceMap.truncated) ?? sources.truncated ?? rawSources.length > MAX_PROJECTED_SOURCE_MAP_SOURCES,
|
|
146
|
+
content_hash: sanitizeSourceMapContentHashValue(sourceMap.content_hash),
|
|
147
|
+
size_bytes: integerValue(sourceMap.size_bytes),
|
|
148
|
+
source_storage_policy: "paths_and_hashes_only",
|
|
149
|
+
projected_source_limit: MAX_PROJECTED_SOURCE_MAP_SOURCES
|
|
150
|
+
});
|
|
151
|
+
return Object.keys(sanitized).length > 0 ? sanitized : null;
|
|
152
|
+
}
|
|
153
|
+
function sanitizeSourceMapArtifactRecord(value) {
|
|
154
|
+
const artifact = objectRecord(value);
|
|
155
|
+
if (Object.keys(artifact).length === 0)
|
|
156
|
+
return {};
|
|
157
|
+
const artifactType = stringValue(artifact.artifact_type) ?? stringValue(artifact.type);
|
|
158
|
+
const path = stringValue(artifact.path);
|
|
159
|
+
const isSourceMapArtifact = artifactType === "source_map" || artifactType === "source-map" || artifactType === "sourcemap" || Boolean(path?.endsWith(".map")) || hasSourceMapTelemetrySignal(artifact);
|
|
160
|
+
const nested = sanitizeSourceMapTelemetry(artifact.source_map);
|
|
161
|
+
const root = isSourceMapArtifact ? sanitizeSourceMapTelemetry(artifact) : null;
|
|
162
|
+
const sourceMap = nested ?? root;
|
|
163
|
+
const output = isSourceMapArtifact ? sourceMapArtifactOutput(artifact) : { ...artifact };
|
|
164
|
+
if (isSourceMapArtifact) {
|
|
165
|
+
for (const key of SOURCE_MAP_ROOT_KEYS)
|
|
166
|
+
output[key] = undefined;
|
|
167
|
+
if ("path" in output)
|
|
168
|
+
output.path = sanitizeSourceMapPathValue(output.path);
|
|
169
|
+
}
|
|
170
|
+
if (sourceMap)
|
|
171
|
+
output.source_map = sourceMap;
|
|
172
|
+
else if ("source_map" in output)
|
|
173
|
+
output.source_map = undefined;
|
|
174
|
+
return output;
|
|
175
|
+
}
|
|
176
|
+
function sourceMapArtifactOutput(artifact) {
|
|
177
|
+
const artifactType = sourceMapArtifactKind(artifact.artifact_type) ?? sourceMapArtifactKind(artifact.type) ?? "source_map";
|
|
178
|
+
return compactObject({
|
|
179
|
+
category: sanitizeSourceMapScalarValue("category", artifact.category),
|
|
180
|
+
scanner: sanitizeSourceMapScalarValue("scanner", artifact.scanner),
|
|
181
|
+
run_type: sanitizeSourceMapScalarValue("run_type", artifact.run_type),
|
|
182
|
+
tool: sanitizeSourceMapScalarValue("tool", artifact.tool),
|
|
183
|
+
package_manager: sanitizeSourceMapScalarValue("package_manager", artifact.package_manager),
|
|
184
|
+
framework: sanitizeSourceMapScalarValue("framework", artifact.framework),
|
|
185
|
+
script: sanitizeSourceMapScalarValue("script", artifact.script),
|
|
186
|
+
artifact_id: sanitizeSourceMapIdentifierValue(artifact.artifact_id),
|
|
187
|
+
artifact_type: artifactType,
|
|
188
|
+
type: sourceMapArtifactKind(artifact.type) ?? sanitizeSourceMapScalarValue("type", artifact.type),
|
|
189
|
+
path: sanitizeSourceMapPathValue(artifact.path),
|
|
190
|
+
content_hash: sanitizeSourceMapContentHashValue(artifact.content_hash),
|
|
191
|
+
size_bytes: integerValue(artifact.size_bytes),
|
|
192
|
+
changed: sanitizeSourceMapScalarValue("changed", artifact.changed),
|
|
193
|
+
mtime_ms: numberValue(artifact.mtime_ms),
|
|
194
|
+
truncated: booleanValue(artifact.truncated)
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
function sanitizeSourceMapPathValue(value) {
|
|
198
|
+
if (typeof value !== "string")
|
|
199
|
+
return null;
|
|
200
|
+
const normalized = value.split("\\").join("/");
|
|
201
|
+
if (!normalized || normalized.includes("\x00"))
|
|
202
|
+
return null;
|
|
203
|
+
if (normalized.startsWith("[source-map-")) {
|
|
204
|
+
if (/^\[source-map-(host_path|unsafe_relative|unsafe_marker):[a-f0-9]{16}\]$/.test(normalized))
|
|
205
|
+
return normalized;
|
|
206
|
+
return pathHashMarker("unsafe_marker", normalized);
|
|
207
|
+
}
|
|
208
|
+
if (isHostPathLike(normalized))
|
|
209
|
+
return pathHashMarker("host_path", normalized);
|
|
210
|
+
const parts = [];
|
|
211
|
+
for (const part of normalized.split("/")) {
|
|
212
|
+
if (!part || part === ".")
|
|
213
|
+
continue;
|
|
214
|
+
if (part === "..")
|
|
215
|
+
return pathHashMarker("unsafe_relative", normalized);
|
|
216
|
+
parts.push(part);
|
|
217
|
+
}
|
|
218
|
+
if (parts.length === 0)
|
|
219
|
+
return null;
|
|
220
|
+
return truncatedString(parts.join("/"), MAX_SOURCE_MAP_STRING);
|
|
221
|
+
}
|
|
222
|
+
function sanitizeSourceMapIdentifierValue(value) {
|
|
223
|
+
if (typeof value !== "string")
|
|
224
|
+
return null;
|
|
225
|
+
const normalized = value.split("\\").join("/");
|
|
226
|
+
if (!normalized || normalized.includes("\x00"))
|
|
227
|
+
return null;
|
|
228
|
+
if (normalized.startsWith("[source-map-")) {
|
|
229
|
+
if (/^\[source-map-id:[a-f0-9]{16}\]$/.test(normalized))
|
|
230
|
+
return normalized;
|
|
231
|
+
return identifierHashMarker(normalized);
|
|
232
|
+
}
|
|
233
|
+
if (isHostPathLike(normalized) || normalized.includes("/") || normalized.includes("..") || /\s/.test(normalized))
|
|
234
|
+
return identifierHashMarker(normalized);
|
|
235
|
+
return truncatedString(normalized, MAX_SOURCE_MAP_STRING);
|
|
236
|
+
}
|
|
237
|
+
function sourceMapFallbackIdentifier(value) {
|
|
238
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
239
|
+
return identifierHashMarker(`fallback:${text ?? String(value)}`);
|
|
240
|
+
}
|
|
241
|
+
function sanitizeSourceMapContextRecord(value) {
|
|
242
|
+
const record = objectRecord(value);
|
|
243
|
+
const output = { ...record };
|
|
244
|
+
sanitizeContextId(output, "artifact_id");
|
|
245
|
+
sanitizeContextId(output, "source_map_id");
|
|
246
|
+
sanitizeContextId(output, "source_map_artifact_id");
|
|
247
|
+
sanitizeContextId(output, "javascript_artifact_id");
|
|
248
|
+
sanitizeContextPath(output, "path");
|
|
249
|
+
sanitizeContextPath(output, "source_map_path");
|
|
250
|
+
sanitizeContextPath(output, "javascript_path");
|
|
251
|
+
sanitizeContextPath(output, "file");
|
|
252
|
+
sanitizeContextPath(output, "sourceRoot");
|
|
253
|
+
sanitizeContextPath(output, "source_root");
|
|
254
|
+
sanitizeContextContentHash(output, "content_hash");
|
|
255
|
+
sanitizeContextArtifactType(output, "artifact_type");
|
|
256
|
+
sanitizeContextArtifactType(output, "type");
|
|
257
|
+
sanitizeContextScalar(output, "category", "category");
|
|
258
|
+
sanitizeContextScalar(output, "scanner", "scanner");
|
|
259
|
+
sanitizeContextScalar(output, "run_type", "run_type");
|
|
260
|
+
sanitizeContextScalar(output, "tool", "tool");
|
|
261
|
+
sanitizeContextScalar(output, "package_manager", "package_manager");
|
|
262
|
+
sanitizeContextScalar(output, "framework", "framework");
|
|
263
|
+
sanitizeContextScalar(output, "script", "script");
|
|
264
|
+
sanitizeContextScalar(output, "changed", "changed");
|
|
265
|
+
return output;
|
|
266
|
+
}
|
|
267
|
+
function sourceMapSourceRowId(sourceMapId, ordinal) {
|
|
268
|
+
return `srcmap_source_${createHash2("md5").update(sourceMapId).update(":").update(String(ordinal)).digest("hex")}`;
|
|
269
|
+
}
|
|
270
|
+
function upsertSourceMapProjection(db, event, index) {
|
|
271
|
+
const sourceMap = sourceMapObject(event, index);
|
|
272
|
+
if (!sourceMap)
|
|
273
|
+
return;
|
|
274
|
+
const body = objectRecord(event.body);
|
|
275
|
+
const rootArtifact = sanitizeSourceMapArtifactRecord(body);
|
|
276
|
+
const artifact = compactObject({
|
|
277
|
+
...rootArtifact,
|
|
278
|
+
...objectRecord(event.body?.artifact)
|
|
279
|
+
});
|
|
280
|
+
const attrs = objectRecord(event.attributes);
|
|
281
|
+
const metadata = objectRecord(index.metadata);
|
|
282
|
+
const sourceMapIdCandidate = firstString([sourceMap, artifact, attrs, metadata], ["source_map_id", "source_map_artifact_id", "artifact_id"]) ?? index.artifact_id;
|
|
283
|
+
const sourceMapId = sanitizeSourceMapIdentifierValue(sourceMapIdCandidate) ?? sourceMapFallbackIdentifier(event.event_id);
|
|
284
|
+
const sourceMapArtifactId = sanitizeSourceMapIdentifierValue(firstString([sourceMap, artifact, attrs, metadata], ["source_map_artifact_id", "artifact_id"]) ?? sourceMapId) ?? sourceMapId;
|
|
285
|
+
const sourceMapPath = sanitizeSourceMapPathValue(firstString([sourceMap, artifact, attrs, metadata], ["source_map_path", "path"]));
|
|
286
|
+
const javascriptArtifactId = sanitizeSourceMapIdentifierValue(sourceMap.javascript_artifact_id);
|
|
287
|
+
const contentHash = sanitizeSourceMapContentHashValue(sourceMap.content_hash) ?? sanitizeSourceMapContentHashValue(artifact.content_hash) ?? sanitizeSourceMapContentHashValue(attrs.content_hash);
|
|
288
|
+
const sources = projectedSources(sourceMap);
|
|
289
|
+
const sourceCount = integerValue(sourceMap.source_count) ?? sources.total_count ?? sources.rows.length;
|
|
290
|
+
const namesCount = integerValue(sourceMap.names_count);
|
|
291
|
+
const mappingsLength = integerValue(sourceMap.mappings_length);
|
|
292
|
+
const hasSourcesContent = booleanValue(sourceMap.has_sources_content) ?? sources.rows.some((source) => source.has_content);
|
|
293
|
+
const truncated = booleanValue(sourceMap.truncated) ?? sources.truncated ?? sources.rows.length < sourceCount;
|
|
294
|
+
const projectionMetadata = compactObject({
|
|
295
|
+
category: "source_map",
|
|
296
|
+
source_map_artifact_id: sourceMapArtifactId,
|
|
297
|
+
javascript_artifact_id: javascriptArtifactId,
|
|
298
|
+
source_map_path: sourceMapPath,
|
|
299
|
+
javascript_path: stringValue(sourceMap.javascript_path),
|
|
300
|
+
source_root: stringValue(sourceMap.source_root),
|
|
301
|
+
file: stringValue(sourceMap.file),
|
|
302
|
+
version: integerValue(sourceMap.version),
|
|
303
|
+
validation_status: validationStatus(sourceMap.validation_status),
|
|
304
|
+
validation_error: sanitizeSourceMapValidationError(sourceMap.validation_error),
|
|
305
|
+
source_count: sourceCount,
|
|
306
|
+
names_count: namesCount,
|
|
307
|
+
mappings_length: mappingsLength,
|
|
308
|
+
has_sources_content: hasSourcesContent,
|
|
309
|
+
truncated,
|
|
310
|
+
content_hash: contentHash,
|
|
311
|
+
size_bytes: integerValue(sourceMap.size_bytes) ?? integerValue(artifact.size_bytes) ?? integerValue(attrs.size_bytes),
|
|
312
|
+
source_storage_policy: "paths_and_hashes_only",
|
|
313
|
+
projected_source_limit: MAX_PROJECTED_SOURCE_MAP_SOURCES
|
|
314
|
+
});
|
|
315
|
+
db.prepare(`
|
|
316
|
+
INSERT INTO source_maps (
|
|
317
|
+
id, event_id, project_id, machine_id, repo_id, app_id, process_id, run_id,
|
|
318
|
+
environment, source_map_artifact_id, javascript_artifact_id, source_map_path,
|
|
319
|
+
javascript_path, source_root, file, version, validation_status, validation_error,
|
|
320
|
+
source_count, names_count, mappings_length, has_sources_content, truncated,
|
|
321
|
+
content_hash, size_bytes, metadata
|
|
322
|
+
)
|
|
323
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
324
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
325
|
+
event_id = COALESCE(excluded.event_id, source_maps.event_id),
|
|
326
|
+
project_id = COALESCE(excluded.project_id, source_maps.project_id),
|
|
327
|
+
machine_id = COALESCE(excluded.machine_id, source_maps.machine_id),
|
|
328
|
+
repo_id = COALESCE(excluded.repo_id, source_maps.repo_id),
|
|
329
|
+
app_id = COALESCE(excluded.app_id, source_maps.app_id),
|
|
330
|
+
process_id = COALESCE(excluded.process_id, source_maps.process_id),
|
|
331
|
+
run_id = COALESCE(excluded.run_id, source_maps.run_id),
|
|
332
|
+
environment = COALESCE(excluded.environment, source_maps.environment),
|
|
333
|
+
source_map_artifact_id = COALESCE(excluded.source_map_artifact_id, source_maps.source_map_artifact_id),
|
|
334
|
+
javascript_artifact_id = excluded.javascript_artifact_id,
|
|
335
|
+
source_map_path = COALESCE(excluded.source_map_path, source_maps.source_map_path),
|
|
336
|
+
javascript_path = excluded.javascript_path,
|
|
337
|
+
source_root = excluded.source_root,
|
|
338
|
+
file = excluded.file,
|
|
339
|
+
version = excluded.version,
|
|
340
|
+
validation_status = excluded.validation_status,
|
|
341
|
+
validation_error = excluded.validation_error,
|
|
342
|
+
source_count = excluded.source_count,
|
|
343
|
+
names_count = excluded.names_count,
|
|
344
|
+
mappings_length = excluded.mappings_length,
|
|
345
|
+
has_sources_content = excluded.has_sources_content,
|
|
346
|
+
truncated = excluded.truncated,
|
|
347
|
+
content_hash = COALESCE(excluded.content_hash, source_maps.content_hash),
|
|
348
|
+
size_bytes = COALESCE(excluded.size_bytes, source_maps.size_bytes),
|
|
349
|
+
metadata = excluded.metadata
|
|
350
|
+
`).run(sourceMapId, event.event_id, index.project_id ?? null, index.machine_id ?? event.machine_id ?? null, index.repo_id ?? event.repo_id ?? null, index.app_id ?? event.app_id ?? null, index.process_id ?? event.process_id ?? null, index.run_id ?? event.run_id ?? null, index.environment ?? event.environment ?? null, sourceMapArtifactId, javascriptArtifactId, sourceMapPath, stringValue(sourceMap.javascript_path), stringValue(sourceMap.source_root), stringValue(sourceMap.file), integerValue(sourceMap.version), validationStatus(sourceMap.validation_status), sanitizeSourceMapValidationError(sourceMap.validation_error), sourceCount, namesCount, mappingsLength, hasSourcesContent ? 1 : 0, truncated ? 1 : 0, contentHash, integerValue(sourceMap.size_bytes) ?? integerValue(artifact.size_bytes) ?? integerValue(attrs.size_bytes), JSON.stringify(projectionMetadata));
|
|
351
|
+
db.prepare("DELETE FROM source_map_sources WHERE source_map_id = ?").run(sourceMapId);
|
|
352
|
+
const insertSource = db.prepare(`
|
|
353
|
+
INSERT INTO source_map_sources (
|
|
354
|
+
id, source_map_id, ordinal, source_path, has_content, content_hash, metadata
|
|
355
|
+
)
|
|
356
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
357
|
+
`);
|
|
358
|
+
for (const source of sources.rows) {
|
|
359
|
+
insertSource.run(sourceMapSourceRowId(sourceMapId, source.ordinal), sourceMapId, source.ordinal, source.source_path, source.has_content ? 1 : 0, source.content_hash, JSON.stringify({
|
|
360
|
+
source_storage_policy: "paths_and_hashes_only",
|
|
361
|
+
truncated: sources.truncated
|
|
362
|
+
}));
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
function sourceMapObject(event, index) {
|
|
366
|
+
const body = objectRecord(event.body);
|
|
367
|
+
const artifact = objectRecord(body.artifact);
|
|
368
|
+
const attrs = objectRecord(event.attributes);
|
|
369
|
+
const metadata = objectRecord(index.metadata);
|
|
370
|
+
const sanitizedRootArtifact = sanitizeSourceMapArtifactRecord(body);
|
|
371
|
+
const sanitizedArtifact = sanitizeSourceMapArtifactRecord(artifact);
|
|
372
|
+
const sanitizedAttrs = sanitizeSourceMapArtifactRecord(attrs);
|
|
373
|
+
const candidates = [
|
|
374
|
+
objectRecord(artifact.source_map),
|
|
375
|
+
objectRecord(body.source_map),
|
|
376
|
+
objectRecord(attrs.source_map),
|
|
377
|
+
objectRecord(metadata.source_map),
|
|
378
|
+
objectRecord(sanitizedRootArtifact.source_map),
|
|
379
|
+
objectRecord(sanitizedArtifact.source_map),
|
|
380
|
+
objectRecord(sanitizedAttrs.source_map)
|
|
381
|
+
];
|
|
382
|
+
const sourceMap = candidates.find((candidate) => Object.keys(candidate).length > 0) ?? null;
|
|
383
|
+
return sanitizeSourceMapTelemetry(sourceMap);
|
|
384
|
+
}
|
|
385
|
+
function projectedSources(sourceMap) {
|
|
386
|
+
const rawSources = Array.isArray(sourceMap.sources) ? sourceMap.sources : [];
|
|
387
|
+
const rows = [];
|
|
388
|
+
const usedOrdinals = new Set;
|
|
389
|
+
let nextOrdinal = 0;
|
|
390
|
+
for (const [index, raw] of rawSources.entries()) {
|
|
391
|
+
if (rows.length >= MAX_PROJECTED_SOURCE_MAP_SOURCES)
|
|
392
|
+
break;
|
|
393
|
+
const source = objectRecord(raw);
|
|
394
|
+
if (Object.keys(source).length === 0)
|
|
395
|
+
continue;
|
|
396
|
+
let ordinal = integerValue(source.ordinal) ?? index;
|
|
397
|
+
if (ordinal < 0)
|
|
398
|
+
ordinal = index;
|
|
399
|
+
while (usedOrdinals.has(ordinal))
|
|
400
|
+
ordinal = nextOrdinal++;
|
|
401
|
+
usedOrdinals.add(ordinal);
|
|
402
|
+
nextOrdinal = Math.max(nextOrdinal, ordinal + 1);
|
|
403
|
+
rows.push({
|
|
404
|
+
ordinal,
|
|
405
|
+
source_path: sanitizeSourceMapPathValue(source.source_path),
|
|
406
|
+
has_content: booleanValue(source.has_content) ?? false,
|
|
407
|
+
content_hash: sanitizeSourceMapContentHashValue(source.content_hash)
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
const totalCount = integerValue(sourceMap.source_count);
|
|
411
|
+
return {
|
|
412
|
+
rows,
|
|
413
|
+
total_count: totalCount,
|
|
414
|
+
truncated: booleanValue(sourceMap.truncated) ?? rawSources.length > MAX_PROJECTED_SOURCE_MAP_SOURCES
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
function sanitizedSources(sourceMap) {
|
|
418
|
+
const rawSources = Array.isArray(sourceMap.sources) ? sourceMap.sources : [];
|
|
419
|
+
const rawSourcesContent = Array.isArray(sourceMap.sourcesContent) ? sourceMap.sourcesContent : [];
|
|
420
|
+
const rows = [];
|
|
421
|
+
const usedOrdinals = new Set;
|
|
422
|
+
let nextOrdinal = 0;
|
|
423
|
+
for (const [index, raw] of rawSources.entries()) {
|
|
424
|
+
if (rows.length >= MAX_PROJECTED_SOURCE_MAP_SOURCES)
|
|
425
|
+
break;
|
|
426
|
+
const source = objectRecord(raw);
|
|
427
|
+
const content = rawSourcesContent[index] ?? (typeof source.content === "string" ? source.content : undefined);
|
|
428
|
+
let ordinal = integerValue(source.ordinal) ?? index;
|
|
429
|
+
if (ordinal < 0)
|
|
430
|
+
ordinal = index;
|
|
431
|
+
while (usedOrdinals.has(ordinal))
|
|
432
|
+
ordinal = nextOrdinal++;
|
|
433
|
+
usedOrdinals.add(ordinal);
|
|
434
|
+
nextOrdinal = Math.max(nextOrdinal, ordinal + 1);
|
|
435
|
+
const computedContentHash = typeof content === "string" ? createHash2("sha256").update(content).digest("hex") : null;
|
|
436
|
+
rows.push({
|
|
437
|
+
ordinal,
|
|
438
|
+
source_path: sanitizeSourceMapPathValue(typeof raw === "string" ? raw : source.source_path ?? source.path ?? source.source),
|
|
439
|
+
has_content: booleanValue(source.has_content) ?? typeof content === "string",
|
|
440
|
+
content_hash: computedContentHash ?? sanitizeSourceMapContentHashValue(source.content_hash)
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
return {
|
|
444
|
+
rows,
|
|
445
|
+
truncated: rawSources.length > MAX_PROJECTED_SOURCE_MAP_SOURCES
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
function validationStatus(value) {
|
|
449
|
+
const status = stringValue(value);
|
|
450
|
+
if (status === "parsed" || status === "malformed" || status === "too_large" || status === "unsupported")
|
|
451
|
+
return status;
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
function linkedBy(value) {
|
|
455
|
+
const linked = stringValue(value);
|
|
456
|
+
if (linked === "adjacent_path" || linked === "file_field" || linked === "none")
|
|
457
|
+
return linked;
|
|
458
|
+
return null;
|
|
459
|
+
}
|
|
460
|
+
function hasSourceMapTelemetrySignal(sourceMap) {
|
|
461
|
+
const artifactType = stringValue(sourceMap.artifact_type) ?? stringValue(sourceMap.type);
|
|
462
|
+
const path = stringValue(sourceMap.path);
|
|
463
|
+
if (artifactType === "source_map" || artifactType === "source-map" || artifactType === "sourcemap" || Boolean(path?.endsWith(".map")))
|
|
464
|
+
return true;
|
|
465
|
+
for (const key of SOURCE_MAP_ROOT_KEYS) {
|
|
466
|
+
if (key === "source_storage_policy" || key === "projected_source_limit")
|
|
467
|
+
continue;
|
|
468
|
+
const value = sourceMap[key];
|
|
469
|
+
if (value === null || value === undefined)
|
|
470
|
+
continue;
|
|
471
|
+
if (Array.isArray(value)) {
|
|
472
|
+
if (value.length > 0)
|
|
473
|
+
return true;
|
|
474
|
+
continue;
|
|
475
|
+
}
|
|
476
|
+
if (typeof value === "string") {
|
|
477
|
+
if (value.length > 0)
|
|
478
|
+
return true;
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
if (typeof value === "boolean") {
|
|
482
|
+
if (value)
|
|
483
|
+
return true;
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
return true;
|
|
487
|
+
}
|
|
488
|
+
return false;
|
|
489
|
+
}
|
|
490
|
+
function sanitizeSourceMapValidationError(value) {
|
|
491
|
+
if (typeof value !== "string")
|
|
492
|
+
return null;
|
|
493
|
+
if (SAFE_VALIDATION_ERRORS.has(value) || /^\[source-map-validation-error:[a-f0-9]{16}\]$/.test(value))
|
|
494
|
+
return truncatedString(value, MAX_SOURCE_MAP_ERROR);
|
|
495
|
+
return `[source-map-validation-error:${createHash2("sha256").update(value).digest("hex").slice(0, 16)}]`;
|
|
496
|
+
}
|
|
497
|
+
function firstString(records, keys) {
|
|
498
|
+
for (const key of keys) {
|
|
499
|
+
for (const record of records) {
|
|
500
|
+
const value = stringValue(record[key]);
|
|
501
|
+
if (value)
|
|
502
|
+
return value;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
return null;
|
|
506
|
+
}
|
|
507
|
+
function objectRecord(value) {
|
|
508
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
509
|
+
}
|
|
510
|
+
function stringValue(value) {
|
|
511
|
+
if (typeof value !== "string")
|
|
512
|
+
return null;
|
|
513
|
+
return truncatedString(value, MAX_SOURCE_MAP_STRING);
|
|
514
|
+
}
|
|
515
|
+
function truncatedString(value, maxLength) {
|
|
516
|
+
if (typeof value !== "string")
|
|
517
|
+
return null;
|
|
518
|
+
if (value.length <= maxLength)
|
|
519
|
+
return value;
|
|
520
|
+
return `${value.slice(0, maxLength)}... [truncated]`;
|
|
521
|
+
}
|
|
522
|
+
function integerValue(value) {
|
|
523
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
524
|
+
return null;
|
|
525
|
+
return Math.trunc(value);
|
|
526
|
+
}
|
|
527
|
+
function numberValue(value) {
|
|
528
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
529
|
+
}
|
|
530
|
+
function booleanValue(value) {
|
|
531
|
+
return typeof value === "boolean" ? value : null;
|
|
532
|
+
}
|
|
533
|
+
function isHostPathLike(value) {
|
|
534
|
+
return value.startsWith("/") || value.startsWith("~/") || value.startsWith("//") || /^[a-zA-Z]:\//.test(value) || /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
|
|
535
|
+
}
|
|
536
|
+
function pathHashMarker(kind, value) {
|
|
537
|
+
return `[source-map-${kind}:${createHash2("sha256").update(value).digest("hex").slice(0, 16)}]`;
|
|
538
|
+
}
|
|
539
|
+
function identifierHashMarker(value) {
|
|
540
|
+
return `[source-map-id:${createHash2("sha256").update(value).digest("hex").slice(0, 16)}]`;
|
|
541
|
+
}
|
|
542
|
+
function scalarHashMarker(value) {
|
|
543
|
+
return `[source-map-scalar:${createHash2("sha256").update(value).digest("hex").slice(0, 16)}]`;
|
|
544
|
+
}
|
|
545
|
+
var SOURCE_MAP_SCALAR_ALLOWLISTS = {
|
|
546
|
+
category: new Set(["build_artifact", "source_map"]),
|
|
547
|
+
scanner: new Set(["common-output-roots"]),
|
|
548
|
+
run_type: new Set(["command", "build", "test", "dev-server"]),
|
|
549
|
+
tool: new Set([
|
|
550
|
+
"ava",
|
|
551
|
+
"biome",
|
|
552
|
+
"build",
|
|
553
|
+
"eslint",
|
|
554
|
+
"jest",
|
|
555
|
+
"mocha",
|
|
556
|
+
"next",
|
|
557
|
+
"playwright",
|
|
558
|
+
"rollup",
|
|
559
|
+
"svelte-kit",
|
|
560
|
+
"test",
|
|
561
|
+
"tsc",
|
|
562
|
+
"turbo",
|
|
563
|
+
"vite",
|
|
564
|
+
"vitest",
|
|
565
|
+
"webpack"
|
|
566
|
+
]),
|
|
567
|
+
package_manager: new Set(["bun", "npm", "pnpm", "yarn"]),
|
|
568
|
+
framework: new Set(["astro", "next", "nuxt", "remix", "svelte-kit", "vite"]),
|
|
569
|
+
script: new Set([
|
|
570
|
+
"build",
|
|
571
|
+
"check",
|
|
572
|
+
"dev",
|
|
573
|
+
"lint",
|
|
574
|
+
"serve",
|
|
575
|
+
"start",
|
|
576
|
+
"test",
|
|
577
|
+
"typecheck"
|
|
578
|
+
]),
|
|
579
|
+
type: new Set(["source_map"]),
|
|
580
|
+
changed: new Set(["created", "modified"])
|
|
581
|
+
};
|
|
582
|
+
function sanitizeSourceMapScalarValue(field, value) {
|
|
583
|
+
if (typeof value !== "string")
|
|
584
|
+
return null;
|
|
585
|
+
const normalized = value.trim().split("\\").join("/");
|
|
586
|
+
if (!normalized || normalized.includes("\x00"))
|
|
587
|
+
return null;
|
|
588
|
+
if (normalized.startsWith("[source-map-")) {
|
|
589
|
+
if (/^\[source-map-scalar:[a-f0-9]{16}\]$/.test(normalized))
|
|
590
|
+
return normalized;
|
|
591
|
+
return scalarHashMarker(normalized);
|
|
592
|
+
}
|
|
593
|
+
if (SOURCE_MAP_SCALAR_ALLOWLISTS[field]?.has(normalized))
|
|
594
|
+
return normalized;
|
|
595
|
+
if (isHostPathLike(normalized) || normalized.includes("/") || normalized.includes("..") || /\s/.test(normalized) || !/^[a-zA-Z0-9_.-]+$/.test(normalized))
|
|
596
|
+
return scalarHashMarker(normalized);
|
|
597
|
+
return scalarHashMarker(normalized);
|
|
598
|
+
}
|
|
599
|
+
function sourceMapArtifactKind(value) {
|
|
600
|
+
const kind = stringValue(value);
|
|
601
|
+
if (kind === "source_map" || kind === "source-map" || kind === "sourcemap") {
|
|
602
|
+
return "source_map";
|
|
603
|
+
}
|
|
604
|
+
return null;
|
|
605
|
+
}
|
|
606
|
+
function sanitizeSourceMapContentHashValue(value) {
|
|
607
|
+
if (typeof value !== "string")
|
|
608
|
+
return null;
|
|
609
|
+
const normalized = value.trim();
|
|
610
|
+
if (!normalized || normalized.includes("\x00"))
|
|
611
|
+
return null;
|
|
612
|
+
if (/^\[source-map-content-hash:[a-f0-9]{16}\]$/.test(normalized)) {
|
|
613
|
+
return normalized;
|
|
614
|
+
}
|
|
615
|
+
if (/^[a-f0-9]{64}$/i.test(normalized))
|
|
616
|
+
return normalized.toLowerCase();
|
|
617
|
+
if (/^sha256:[a-f0-9]{64}$/i.test(normalized)) {
|
|
618
|
+
return normalized.toLowerCase();
|
|
619
|
+
}
|
|
620
|
+
return `[source-map-content-hash:${createHash2("sha256").update(normalized).digest("hex").slice(0, 16)}]`;
|
|
621
|
+
}
|
|
622
|
+
function sanitizeContextId(record, key) {
|
|
623
|
+
if (!(key in record))
|
|
624
|
+
return;
|
|
625
|
+
const value = sanitizeSourceMapIdentifierValue(record[key]);
|
|
626
|
+
if (value)
|
|
627
|
+
record[key] = value;
|
|
628
|
+
else
|
|
629
|
+
record[key] = undefined;
|
|
630
|
+
}
|
|
631
|
+
function sanitizeContextPath(record, key) {
|
|
632
|
+
if (!(key in record))
|
|
633
|
+
return;
|
|
634
|
+
const value = sanitizeSourceMapPathValue(record[key]);
|
|
635
|
+
if (value)
|
|
636
|
+
record[key] = value;
|
|
637
|
+
else
|
|
638
|
+
record[key] = undefined;
|
|
639
|
+
}
|
|
640
|
+
function sanitizeContextContentHash(record, key) {
|
|
641
|
+
if (!(key in record))
|
|
642
|
+
return;
|
|
643
|
+
const value = sanitizeSourceMapContentHashValue(record[key]);
|
|
644
|
+
if (value)
|
|
645
|
+
record[key] = value;
|
|
646
|
+
else
|
|
647
|
+
record[key] = undefined;
|
|
648
|
+
}
|
|
649
|
+
function sanitizeContextArtifactType(record, key) {
|
|
650
|
+
if (!(key in record))
|
|
651
|
+
return;
|
|
652
|
+
const value = sourceMapArtifactKind(record[key]);
|
|
653
|
+
if (value)
|
|
654
|
+
record[key] = value;
|
|
655
|
+
else
|
|
656
|
+
record[key] = undefined;
|
|
657
|
+
}
|
|
658
|
+
function sanitizeContextScalar(record, key, field) {
|
|
659
|
+
if (!(key in record))
|
|
660
|
+
return;
|
|
661
|
+
const value = sanitizeSourceMapScalarValue(field, record[key]);
|
|
662
|
+
if (value)
|
|
663
|
+
record[key] = value;
|
|
664
|
+
else
|
|
665
|
+
record[key] = undefined;
|
|
666
|
+
}
|
|
667
|
+
function compactObject(input) {
|
|
668
|
+
const output = {};
|
|
669
|
+
for (const [key, value] of Object.entries(input)) {
|
|
670
|
+
if (value !== null && value !== undefined)
|
|
671
|
+
output[key] = value;
|
|
672
|
+
}
|
|
673
|
+
return output;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// src/lib/test-report-projections.ts
|
|
677
|
+
import { createHash as createHash3 } from "crypto";
|
|
678
|
+
var MAX_PROJECTED_TEST_REPORT_SUITES = 20;
|
|
679
|
+
var MAX_PROJECTED_TEST_REPORT_CASES = 50;
|
|
680
|
+
var MAX_PROJECTED_PARSE_ERROR_LENGTH = 500;
|
|
681
|
+
var REDACTED_PARSE_ERROR = "[redacted raw test-report parse error]";
|
|
682
|
+
function sanitizedTestReportMetadata(reportInput, metadataInput, attributesInput, extraInput) {
|
|
683
|
+
const report = objectRecord2(reportInput);
|
|
684
|
+
const metadata = metadataInput ?? {};
|
|
685
|
+
const attrs = attributesInput ?? {};
|
|
686
|
+
const redaction = objectRecord2(metadata.redaction);
|
|
687
|
+
return compactObject2({
|
|
688
|
+
category: firstString2(attrs, metadata, "category") ?? "test_report",
|
|
689
|
+
scanner: firstString2(attrs, metadata, "scanner"),
|
|
690
|
+
run_type: firstString2(attrs, metadata, "run_type"),
|
|
691
|
+
tool: firstString2(attrs, metadata, "tool"),
|
|
692
|
+
package_manager: firstString2(attrs, metadata, "package_manager"),
|
|
693
|
+
framework: firstString2(attrs, metadata, "framework"),
|
|
694
|
+
script: firstString2(attrs, metadata, "script"),
|
|
695
|
+
report_id: stringValue2(report.report_id) ?? firstString2(attrs, metadata, "report_id"),
|
|
696
|
+
report_format: stringValue2(report.format) ?? firstString2(attrs, metadata, "report_format") ?? firstString2(attrs, metadata, "format"),
|
|
697
|
+
parser: firstString2(report, attrs, metadata, "parser"),
|
|
698
|
+
parse_status: firstString2(report, attrs, metadata, "parse_status"),
|
|
699
|
+
parse_error: sanitizedParseError(firstString2(report, attrs, metadata, "parse_error")),
|
|
700
|
+
path: firstString2(report, attrs, metadata, "path"),
|
|
701
|
+
size_bytes: firstNumber(report, attrs, metadata, "size_bytes"),
|
|
702
|
+
content_hash: firstString2(report, attrs, metadata, "content_hash"),
|
|
703
|
+
changed: firstString2(report, attrs, metadata, "changed"),
|
|
704
|
+
mtime_ms: firstNumber(report, attrs, metadata, "mtime_ms"),
|
|
705
|
+
tests: firstNumber(report, attrs, metadata, "tests"),
|
|
706
|
+
failures: firstNumber(report, attrs, metadata, "failures"),
|
|
707
|
+
errors: firstNumber(report, attrs, metadata, "errors"),
|
|
708
|
+
skipped: firstNumber(report, attrs, metadata, "skipped"),
|
|
709
|
+
time_seconds: firstNumber(report, attrs, metadata, "time_seconds"),
|
|
710
|
+
suite_count: firstNumber(report, attrs, metadata, "suite_count"),
|
|
711
|
+
testcase_count: firstNumber(report, attrs, metadata, "testcase_count"),
|
|
712
|
+
truncated: firstBoolean(report, attrs, metadata, "truncated") ?? undefined,
|
|
713
|
+
case_storage_policy: "bounded_raw_cases",
|
|
714
|
+
...compactObject2(extraInput ?? {}),
|
|
715
|
+
redaction: Object.keys(redaction).length > 0 ? redaction : undefined
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
function upsertTestReportProjection(db, envelope, index) {
|
|
719
|
+
const attrs = objectRecord2(envelope.attributes);
|
|
720
|
+
const body = objectRecord2(envelope.body);
|
|
721
|
+
const report = objectRecord2(body.test_report);
|
|
722
|
+
const metadata = index.metadata ?? {};
|
|
723
|
+
if (Object.keys(report).length === 0 && !isTestReportCategory(attrs))
|
|
724
|
+
return;
|
|
725
|
+
const reportId = stringValue2(report.report_id) ?? stringValue2(attrs.report_id) ?? stringValue2(metadata.report_id) ?? index.event_id ?? envelope.event_id;
|
|
726
|
+
const suites = reportSuites(report, metadata);
|
|
727
|
+
const projected = projectedCases(reportId, suites);
|
|
728
|
+
const cases = projected.cases;
|
|
729
|
+
const projectionMetadata = sanitizedTestReportMetadata(report, metadata, attrs, {
|
|
730
|
+
projected_case_count: cases.length,
|
|
731
|
+
projected_case_limit: MAX_PROJECTED_TEST_REPORT_CASES
|
|
732
|
+
});
|
|
733
|
+
db.prepare(`
|
|
734
|
+
INSERT INTO test_reports (
|
|
735
|
+
id, event_id, source_event_id, project_id, machine_id, repo_id, app_id,
|
|
736
|
+
process_id, run_id, environment, source, event_time, path, format, parser,
|
|
737
|
+
parse_status, parse_error, size_bytes, content_hash, changed, mtime_ms,
|
|
738
|
+
tests, failures, errors, skipped, time_seconds, suite_count,
|
|
739
|
+
testcase_count, case_stored_count, truncated, metadata
|
|
740
|
+
)
|
|
741
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
742
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
743
|
+
event_id = excluded.event_id,
|
|
744
|
+
source_event_id = excluded.source_event_id,
|
|
745
|
+
project_id = COALESCE(excluded.project_id, test_reports.project_id),
|
|
746
|
+
machine_id = COALESCE(excluded.machine_id, test_reports.machine_id),
|
|
747
|
+
repo_id = COALESCE(excluded.repo_id, test_reports.repo_id),
|
|
748
|
+
app_id = COALESCE(excluded.app_id, test_reports.app_id),
|
|
749
|
+
process_id = COALESCE(excluded.process_id, test_reports.process_id),
|
|
750
|
+
run_id = COALESCE(excluded.run_id, test_reports.run_id),
|
|
751
|
+
environment = COALESCE(excluded.environment, test_reports.environment),
|
|
752
|
+
source = COALESCE(excluded.source, test_reports.source),
|
|
753
|
+
event_time = COALESCE(excluded.event_time, test_reports.event_time),
|
|
754
|
+
path = COALESCE(excluded.path, test_reports.path),
|
|
755
|
+
format = COALESCE(excluded.format, test_reports.format),
|
|
756
|
+
parser = COALESCE(excluded.parser, test_reports.parser),
|
|
757
|
+
parse_status = COALESCE(excluded.parse_status, test_reports.parse_status),
|
|
758
|
+
parse_error = COALESCE(excluded.parse_error, test_reports.parse_error),
|
|
759
|
+
size_bytes = COALESCE(excluded.size_bytes, test_reports.size_bytes),
|
|
760
|
+
content_hash = COALESCE(excluded.content_hash, test_reports.content_hash),
|
|
761
|
+
changed = COALESCE(excluded.changed, test_reports.changed),
|
|
762
|
+
mtime_ms = COALESCE(excluded.mtime_ms, test_reports.mtime_ms),
|
|
763
|
+
tests = COALESCE(excluded.tests, test_reports.tests),
|
|
764
|
+
failures = COALESCE(excluded.failures, test_reports.failures),
|
|
765
|
+
errors = COALESCE(excluded.errors, test_reports.errors),
|
|
766
|
+
skipped = COALESCE(excluded.skipped, test_reports.skipped),
|
|
767
|
+
time_seconds = COALESCE(excluded.time_seconds, test_reports.time_seconds),
|
|
768
|
+
suite_count = COALESCE(excluded.suite_count, test_reports.suite_count),
|
|
769
|
+
testcase_count = COALESCE(excluded.testcase_count, test_reports.testcase_count),
|
|
770
|
+
case_stored_count = excluded.case_stored_count,
|
|
771
|
+
truncated = excluded.truncated,
|
|
772
|
+
metadata = excluded.metadata
|
|
773
|
+
`).run(reportId, envelope.event_id, index.source_event_id ?? envelope.source_event_id ?? null, index.project_id ?? null, index.machine_id ?? null, index.repo_id ?? null, index.app_id ?? null, index.process_id ?? null, index.run_id ?? null, index.environment ?? null, envelope.source, envelope.event_time, stringValue2(report.path) ?? stringValue2(attrs.path) ?? stringValue2(metadata.path), stringValue2(report.format) ?? stringValue2(attrs.report_format) ?? stringValue2(metadata.report_format), stringValue2(report.parser) ?? stringValue2(attrs.parser) ?? stringValue2(metadata.parser), stringValue2(report.parse_status) ?? stringValue2(attrs.parse_status) ?? stringValue2(metadata.parse_status), sanitizedParseError(stringValue2(report.parse_error) ?? stringValue2(attrs.parse_error) ?? stringValue2(metadata.parse_error)), numberValue2(report.size_bytes) ?? numberValue2(attrs.size_bytes) ?? numberValue2(metadata.size_bytes), stringValue2(report.content_hash) ?? stringValue2(attrs.content_hash) ?? stringValue2(metadata.content_hash), stringValue2(report.changed) ?? stringValue2(attrs.changed) ?? stringValue2(metadata.changed), numberValue2(report.mtime_ms) ?? numberValue2(attrs.mtime_ms) ?? numberValue2(metadata.mtime_ms), numberValue2(report.tests) ?? numberValue2(attrs.tests) ?? numberValue2(metadata.tests), numberValue2(report.failures) ?? numberValue2(attrs.failures) ?? numberValue2(metadata.failures), numberValue2(report.errors) ?? numberValue2(attrs.errors) ?? numberValue2(metadata.errors), numberValue2(report.skipped) ?? numberValue2(attrs.skipped) ?? numberValue2(metadata.skipped), numberValue2(report.time_seconds) ?? numberValue2(attrs.time_seconds) ?? numberValue2(metadata.time_seconds), numberValue2(report.suite_count) ?? numberValue2(attrs.suite_count) ?? numberValue2(metadata.suite_count), numberValue2(report.testcase_count) ?? numberValue2(attrs.testcase_count) ?? numberValue2(metadata.testcase_count), cases.length, projected.truncated || boolValue(report.truncated) || boolValue(attrs.truncated) || boolValue(metadata.truncated) ? 1 : 0, JSON.stringify(projectionMetadata));
|
|
774
|
+
db.prepare("DELETE FROM test_cases WHERE report_id = ?").run(reportId);
|
|
775
|
+
const insertCase = db.prepare(`
|
|
776
|
+
INSERT INTO test_cases (
|
|
777
|
+
id, report_id, event_id, project_id, machine_id, repo_id, app_id,
|
|
778
|
+
process_id, run_id, environment, suite_name, suite_index, case_index,
|
|
779
|
+
name, classname, file, status, time_seconds, metadata
|
|
780
|
+
)
|
|
781
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
782
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
783
|
+
report_id = excluded.report_id,
|
|
784
|
+
event_id = excluded.event_id,
|
|
785
|
+
project_id = COALESCE(excluded.project_id, test_cases.project_id),
|
|
786
|
+
machine_id = COALESCE(excluded.machine_id, test_cases.machine_id),
|
|
787
|
+
repo_id = COALESCE(excluded.repo_id, test_cases.repo_id),
|
|
788
|
+
app_id = COALESCE(excluded.app_id, test_cases.app_id),
|
|
789
|
+
process_id = COALESCE(excluded.process_id, test_cases.process_id),
|
|
790
|
+
run_id = COALESCE(excluded.run_id, test_cases.run_id),
|
|
791
|
+
environment = COALESCE(excluded.environment, test_cases.environment),
|
|
792
|
+
suite_name = excluded.suite_name,
|
|
793
|
+
suite_index = excluded.suite_index,
|
|
794
|
+
case_index = excluded.case_index,
|
|
795
|
+
name = excluded.name,
|
|
796
|
+
classname = excluded.classname,
|
|
797
|
+
file = excluded.file,
|
|
798
|
+
status = excluded.status,
|
|
799
|
+
time_seconds = excluded.time_seconds,
|
|
800
|
+
metadata = excluded.metadata
|
|
801
|
+
`);
|
|
802
|
+
for (const testcase of cases) {
|
|
803
|
+
insertCase.run(testcase.id, testcase.report_id, envelope.event_id, index.project_id ?? null, index.machine_id ?? null, index.repo_id ?? null, index.app_id ?? null, index.process_id ?? null, index.run_id ?? null, index.environment ?? null, testcase.suite_name, testcase.suite_index, testcase.case_index, testcase.name, testcase.classname, testcase.file, testcase.status, testcase.time_seconds, JSON.stringify(testcase.metadata));
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
function projectedCases(reportId, suites) {
|
|
807
|
+
const cases = [];
|
|
808
|
+
let truncated = suites.length > MAX_PROJECTED_TEST_REPORT_SUITES;
|
|
809
|
+
suites.slice(0, MAX_PROJECTED_TEST_REPORT_SUITES).forEach((suite, suiteIndex) => {
|
|
810
|
+
const suiteName = stringValue2(suite.name);
|
|
811
|
+
const suiteCases = objectArray(suite.cases);
|
|
812
|
+
const remaining = MAX_PROJECTED_TEST_REPORT_CASES - cases.length;
|
|
813
|
+
if (remaining <= 0) {
|
|
814
|
+
if (suiteCases.length > 0)
|
|
815
|
+
truncated = true;
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
if (suiteCases.length > remaining)
|
|
819
|
+
truncated = true;
|
|
820
|
+
suiteCases.slice(0, remaining).forEach((testcase, caseIndex) => {
|
|
821
|
+
const name = stringValue2(testcase.name);
|
|
822
|
+
const classname = stringValue2(testcase.classname);
|
|
823
|
+
const file = stringValue2(testcase.file);
|
|
824
|
+
const status = stringValue2(testcase.status);
|
|
825
|
+
const timeSeconds = numberValue2(testcase.time_seconds);
|
|
826
|
+
cases.push({
|
|
827
|
+
id: testcaseProjectionId(reportId, suiteIndex, caseIndex),
|
|
828
|
+
report_id: reportId,
|
|
829
|
+
suite_name: suiteName,
|
|
830
|
+
suite_index: suiteIndex,
|
|
831
|
+
case_index: caseIndex,
|
|
832
|
+
name,
|
|
833
|
+
classname,
|
|
834
|
+
file,
|
|
835
|
+
status,
|
|
836
|
+
time_seconds: timeSeconds,
|
|
837
|
+
metadata: sanitizedTestCaseMetadata(testcase, suiteName, suiteIndex, caseIndex)
|
|
838
|
+
});
|
|
839
|
+
});
|
|
840
|
+
});
|
|
841
|
+
return { cases, truncated };
|
|
842
|
+
}
|
|
843
|
+
function sanitizedTestCaseMetadata(testcase, suiteName, suiteIndex, caseIndex) {
|
|
844
|
+
return compactObject2({
|
|
845
|
+
suite_name: suiteName,
|
|
846
|
+
suite_index: suiteIndex,
|
|
847
|
+
case_index: caseIndex,
|
|
848
|
+
name: stringValue2(testcase.name),
|
|
849
|
+
classname: stringValue2(testcase.classname),
|
|
850
|
+
file: stringValue2(testcase.file),
|
|
851
|
+
status: stringValue2(testcase.status),
|
|
852
|
+
time_seconds: numberValue2(testcase.time_seconds),
|
|
853
|
+
case_storage_policy: "bounded_raw_cases"
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
function reportSuites(report, metadata) {
|
|
857
|
+
const reportSuites2 = objectArray(report.suites);
|
|
858
|
+
if (reportSuites2.length > 0)
|
|
859
|
+
return reportSuites2;
|
|
860
|
+
return objectArray(metadata.suites);
|
|
861
|
+
}
|
|
862
|
+
function testcaseProjectionId(reportId, suiteIndex, caseIndex) {
|
|
863
|
+
const digest = createHash3("sha256").update(reportId).update("\x00").update(String(suiteIndex)).update("\x00").update(String(caseIndex)).digest("hex").slice(0, 32);
|
|
864
|
+
return `testcase_${digest}`;
|
|
865
|
+
}
|
|
866
|
+
function isTestReportCategory(attrs) {
|
|
867
|
+
return stringValue2(attrs.category) === "test_report";
|
|
868
|
+
}
|
|
869
|
+
function objectRecord2(value) {
|
|
870
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
871
|
+
}
|
|
872
|
+
function objectArray(value) {
|
|
873
|
+
if (!Array.isArray(value))
|
|
874
|
+
return [];
|
|
875
|
+
return value.flatMap((item) => {
|
|
876
|
+
const record = objectRecord2(item);
|
|
877
|
+
return Object.keys(record).length > 0 ? [record] : [];
|
|
878
|
+
});
|
|
879
|
+
}
|
|
880
|
+
function compactObject2(value) {
|
|
881
|
+
return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== null));
|
|
882
|
+
}
|
|
883
|
+
function stringValue2(value) {
|
|
884
|
+
if (typeof value === "string")
|
|
885
|
+
return value;
|
|
886
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
887
|
+
return String(value);
|
|
888
|
+
return null;
|
|
889
|
+
}
|
|
890
|
+
function firstString2(first, second, thirdOrKey, maybeKey) {
|
|
891
|
+
const records = typeof thirdOrKey === "string" ? [first, second] : [first, second, thirdOrKey];
|
|
892
|
+
const key = typeof thirdOrKey === "string" ? thirdOrKey : maybeKey;
|
|
893
|
+
if (!key)
|
|
894
|
+
return null;
|
|
895
|
+
for (const record of records) {
|
|
896
|
+
const value = stringValue2(record[key]);
|
|
897
|
+
if (value !== null)
|
|
898
|
+
return value;
|
|
899
|
+
}
|
|
900
|
+
return null;
|
|
901
|
+
}
|
|
902
|
+
function sanitizedParseError(value) {
|
|
903
|
+
if (value === null)
|
|
904
|
+
return null;
|
|
905
|
+
const trimmed = value.trim();
|
|
906
|
+
if (!trimmed)
|
|
907
|
+
return null;
|
|
908
|
+
if (containsRawTestReportPayload(trimmed))
|
|
909
|
+
return REDACTED_PARSE_ERROR;
|
|
910
|
+
if (trimmed.length <= MAX_PROJECTED_PARSE_ERROR_LENGTH)
|
|
911
|
+
return trimmed;
|
|
912
|
+
return `${trimmed.slice(0, MAX_PROJECTED_PARSE_ERROR_LENGTH)}... [truncated]`;
|
|
913
|
+
}
|
|
914
|
+
function containsRawTestReportPayload(value) {
|
|
915
|
+
return /<\s*(testsuites?|testcase|system-out|system-err|failure|error)\b/i.test(value) || /\b(system-out|system_err|system-err|raw_xml)\b/i.test(value) || /failure body/i.test(value);
|
|
916
|
+
}
|
|
917
|
+
function numberValue2(value) {
|
|
918
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
919
|
+
return value;
|
|
920
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
921
|
+
const parsed = Number(value);
|
|
922
|
+
if (Number.isFinite(parsed))
|
|
923
|
+
return parsed;
|
|
924
|
+
}
|
|
925
|
+
return null;
|
|
926
|
+
}
|
|
927
|
+
function firstNumber(first, second, third, key) {
|
|
928
|
+
return numberValue2(first[key]) ?? numberValue2(second[key]) ?? numberValue2(third[key]);
|
|
929
|
+
}
|
|
930
|
+
function boolValue(value) {
|
|
931
|
+
if (typeof value === "boolean")
|
|
932
|
+
return value;
|
|
933
|
+
if (typeof value === "number")
|
|
934
|
+
return value !== 0;
|
|
935
|
+
if (typeof value === "string")
|
|
936
|
+
return value === "true" || value === "1" || value === "yes";
|
|
937
|
+
return false;
|
|
938
|
+
}
|
|
939
|
+
function firstBoolean(first, second, third, key) {
|
|
940
|
+
for (const record of [first, second, third]) {
|
|
941
|
+
if (record[key] !== undefined && record[key] !== null)
|
|
942
|
+
return boolValue(record[key]);
|
|
943
|
+
}
|
|
944
|
+
return null;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
// src/lib/event-store.ts
|
|
948
|
+
var dbDataDirs = new WeakMap;
|
|
949
|
+
var dbLockDepths = new WeakMap;
|
|
950
|
+
function setEventStoreDataDir(db, dataDir) {
|
|
951
|
+
dbDataDirs.set(db, dataDir);
|
|
952
|
+
}
|
|
953
|
+
function getEventStoreDataDir(db) {
|
|
954
|
+
const mapped = dbDataDirs.get(db);
|
|
955
|
+
if (mapped)
|
|
956
|
+
return mapped;
|
|
957
|
+
const explicit = process.env.HASNA_LOGS_DATA_DIR ?? process.env.LOGS_DATA_DIR;
|
|
958
|
+
if (explicit)
|
|
959
|
+
return explicit;
|
|
960
|
+
return join(process.env.HOME ?? "~", ".hasna", "logs");
|
|
961
|
+
}
|
|
962
|
+
function appendRawEvent(db, event) {
|
|
963
|
+
return withEventStoreLock(db, () => {
|
|
964
|
+
const line = Buffer.from(`${JSON.stringify(event)}
|
|
965
|
+
`, "utf8");
|
|
966
|
+
const candidate = getActiveSegment(db, event.ingest_time, line.byteLength);
|
|
967
|
+
const absolutePath = resolveDataPath(db, candidate.relative_path);
|
|
968
|
+
mkdirSync(dirname(absolutePath), { recursive: true });
|
|
969
|
+
const byteOffset = existsSync(absolutePath) ? statSync(absolutePath).size : 0;
|
|
970
|
+
appendAndFlush(absolutePath, line);
|
|
971
|
+
const byteLength = line.byteLength;
|
|
972
|
+
const currentSize = byteOffset + byteLength;
|
|
973
|
+
const recordHash = sha256(line);
|
|
974
|
+
const segmentHash = shouldHashSegmentOnAppend() ? sha256(readFileSync(absolutePath)) : null;
|
|
975
|
+
db.prepare(`
|
|
976
|
+
INSERT INTO event_segments (
|
|
977
|
+
id, relative_path, manifest_path, byte_length, event_count,
|
|
978
|
+
first_event_time, last_event_time, segment_hash, updated_at
|
|
979
|
+
)
|
|
980
|
+
VALUES (?, ?, ?, ?, 1, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
981
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
982
|
+
byte_length = excluded.byte_length,
|
|
983
|
+
event_count = event_segments.event_count + 1,
|
|
984
|
+
first_event_time = COALESCE(event_segments.first_event_time, excluded.first_event_time),
|
|
985
|
+
last_event_time = excluded.last_event_time,
|
|
986
|
+
segment_hash = excluded.segment_hash,
|
|
987
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
988
|
+
`).run(candidate.id, candidate.relative_path, candidate.manifest_path, currentSize, event.event_time, event.event_time, segmentHash);
|
|
989
|
+
writeSegmentManifest(db, candidate.id);
|
|
990
|
+
return {
|
|
991
|
+
segment_id: candidate.id,
|
|
992
|
+
segment_path: candidate.relative_path,
|
|
993
|
+
manifest_path: candidate.manifest_path,
|
|
994
|
+
byte_offset: byteOffset,
|
|
995
|
+
byte_length: byteLength,
|
|
996
|
+
record_hash: recordHash
|
|
997
|
+
};
|
|
998
|
+
});
|
|
999
|
+
}
|
|
1000
|
+
function indexRawEvent(db, event, write) {
|
|
1001
|
+
const existing = getEventRecord(db, event.event_id);
|
|
1002
|
+
if (existing) {
|
|
1003
|
+
if (matchesRawEventIndex(existing, event, write))
|
|
1004
|
+
return;
|
|
1005
|
+
throw new Error(`Event record already indexed with different raw pointer: ${event.event_id}`);
|
|
1006
|
+
}
|
|
1007
|
+
db.prepare(`
|
|
1008
|
+
INSERT INTO event_records (
|
|
1009
|
+
event_id, schema_version, source_event_id, event_type, event_time, ingest_time,
|
|
1010
|
+
severity, source, project_id, page_id, log_id, trace_id, session_id,
|
|
1011
|
+
machine_id, repo_id, app_id, process_id, run_id, span_id, parent_span_id,
|
|
1012
|
+
release_id, environment, artifact_id, privacy_tier,
|
|
1013
|
+
segment_id, segment_path, byte_offset, byte_length, record_hash,
|
|
1014
|
+
message, metadata
|
|
1015
|
+
)
|
|
1016
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1017
|
+
`).run(event.event_id, event.schema_version ?? 1, event.source_event_id ?? null, event.event_type, event.event_time, event.ingest_time, event.severity ?? null, event.source, event.project_id ?? null, event.page_id ?? null, event.log_id ?? null, event.trace_id ?? null, event.session_id ?? null, event.machine_id ?? null, event.repo_id ?? null, event.app_id ?? null, event.process_id ?? null, event.run_id ?? null, event.span_id ?? null, event.parent_span_id ?? null, event.release_id ?? null, event.environment ?? null, event.artifact_id ?? null, event.privacy_tier ?? null, write.segment_id, write.segment_path, write.byte_offset, write.byte_length, write.record_hash, event.message ?? null, event.metadata ? JSON.stringify(event.metadata) : null);
|
|
1018
|
+
}
|
|
1019
|
+
function matchesRawEventIndex(existing, event, write) {
|
|
1020
|
+
return existing.schema_version === (event.schema_version ?? 1) && existing.source_event_id === (event.source_event_id ?? null) && existing.event_type === event.event_type && existing.event_time === event.event_time && existing.ingest_time === event.ingest_time && existing.source === event.source && existing.segment_id === write.segment_id && existing.segment_path === write.segment_path && existing.byte_offset === write.byte_offset && existing.byte_length === write.byte_length && existing.record_hash === write.record_hash;
|
|
1021
|
+
}
|
|
1022
|
+
function getEventRecord(db, eventId) {
|
|
1023
|
+
const row = db.prepare("SELECT * FROM event_records WHERE event_id = ?").get(eventId);
|
|
1024
|
+
return row ?? null;
|
|
1025
|
+
}
|
|
1026
|
+
function readRawEvent(db, eventOrRecord) {
|
|
1027
|
+
const record = typeof eventOrRecord === "string" ? getEventRecord(db, eventOrRecord) : eventOrRecord;
|
|
1028
|
+
if (!record)
|
|
1029
|
+
return null;
|
|
1030
|
+
const bytes = readFileSync(resolveDataPath(db, record.segment_path));
|
|
1031
|
+
const line = bytes.subarray(record.byte_offset, record.byte_offset + record.byte_length);
|
|
1032
|
+
const actualHash = sha256(line);
|
|
1033
|
+
if (actualHash !== record.record_hash) {
|
|
1034
|
+
throw new Error(`Raw event hash mismatch for ${record.event_id}`);
|
|
1035
|
+
}
|
|
1036
|
+
return JSON.parse(line.toString("utf8"));
|
|
1037
|
+
}
|
|
1038
|
+
function verifyEventStore(db) {
|
|
1039
|
+
const errors = [];
|
|
1040
|
+
const segments = db.prepare("SELECT * FROM event_segments ORDER BY relative_path").all();
|
|
1041
|
+
const records = db.prepare("SELECT * FROM event_records ORDER BY event_time, event_id").all();
|
|
1042
|
+
const segmentsByPath = new Map(segments.map((segment) => [segment.relative_path, segment]));
|
|
1043
|
+
const recordsByEventId = new Map(records.map((record) => [record.event_id, record]));
|
|
1044
|
+
const paths = [
|
|
1045
|
+
...new Set([
|
|
1046
|
+
...segments.map((segment) => segment.relative_path),
|
|
1047
|
+
...listSegmentFiles(db)
|
|
1048
|
+
])
|
|
1049
|
+
].sort();
|
|
1050
|
+
let checkedRawEvents = 0;
|
|
1051
|
+
let unindexedRawEvents = 0;
|
|
1052
|
+
for (const relativePath of paths) {
|
|
1053
|
+
const segment = segmentsByPath.get(relativePath);
|
|
1054
|
+
try {
|
|
1055
|
+
const path = resolveDataPath(db, relativePath);
|
|
1056
|
+
if (!existsSync(path)) {
|
|
1057
|
+
errors.push(`Missing segment file: ${relativePath}`);
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
const scan = scanSegmentEvents(db, relativePath, { strict: false });
|
|
1061
|
+
checkedRawEvents += scan.events.length;
|
|
1062
|
+
errors.push(...scan.errors);
|
|
1063
|
+
if (!segment) {
|
|
1064
|
+
errors.push(`Segment file is not indexed in SQLite: ${relativePath}`);
|
|
1065
|
+
} else {
|
|
1066
|
+
if (scan.byte_length !== segment.byte_length) {
|
|
1067
|
+
errors.push(`Segment size mismatch for ${relativePath}: sqlite=${segment.byte_length} file=${scan.byte_length}`);
|
|
1068
|
+
}
|
|
1069
|
+
if (segment.segment_hash && scan.segment_hash !== segment.segment_hash) {
|
|
1070
|
+
errors.push(`Segment hash mismatch for ${relativePath}`);
|
|
1071
|
+
}
|
|
1072
|
+
if (segment.manifest_path && !existsSync(resolveDataPath(db, segment.manifest_path))) {
|
|
1073
|
+
errors.push(`Missing segment manifest: ${segment.manifest_path}`);
|
|
1074
|
+
}
|
|
1075
|
+
if (scan.events.length !== segment.event_count) {
|
|
1076
|
+
errors.push(`Segment event count mismatch for ${relativePath}: sqlite=${segment.event_count} file=${scan.events.length}`);
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
for (const item of scan.events) {
|
|
1080
|
+
const record = recordsByEventId.get(item.event.event_id);
|
|
1081
|
+
if (!record) {
|
|
1082
|
+
unindexedRawEvents += 1;
|
|
1083
|
+
errors.push(`Raw event is not indexed in SQLite: ${item.event.event_id} at ${relativePath}:${item.write.byte_offset}`);
|
|
1084
|
+
continue;
|
|
1085
|
+
}
|
|
1086
|
+
if (record.segment_path !== item.write.segment_path || record.byte_offset !== item.write.byte_offset || record.byte_length !== item.write.byte_length) {
|
|
1087
|
+
errors.push(`Raw event pointer mismatch for ${item.event.event_id}`);
|
|
1088
|
+
}
|
|
1089
|
+
if (record.record_hash !== item.write.record_hash) {
|
|
1090
|
+
errors.push(`Raw event hash index mismatch for ${item.event.event_id}`);
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
} catch (error) {
|
|
1094
|
+
errors.push(errorMessage(error));
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
for (const record of records) {
|
|
1098
|
+
try {
|
|
1099
|
+
const raw = readRawEvent(db, record);
|
|
1100
|
+
if (raw?.event_id !== record.event_id) {
|
|
1101
|
+
errors.push(`Raw event id mismatch for ${record.event_id}`);
|
|
1102
|
+
}
|
|
1103
|
+
if (raw?.event_time !== record.event_time) {
|
|
1104
|
+
errors.push(`Raw event time mismatch for ${record.event_id}`);
|
|
1105
|
+
}
|
|
1106
|
+
} catch (error) {
|
|
1107
|
+
errors.push(errorMessage(error));
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
return {
|
|
1111
|
+
ok: errors.length === 0,
|
|
1112
|
+
checked_records: records.length,
|
|
1113
|
+
checked_segments: paths.length,
|
|
1114
|
+
checked_raw_events: checkedRawEvents,
|
|
1115
|
+
unindexed_raw_events: unindexedRawEvents,
|
|
1116
|
+
errors
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
function rebuildEventStoreIndex(db) {
|
|
1120
|
+
return withEventStoreLock(db, () => rebuildEventStoreIndexLocked(db));
|
|
1121
|
+
}
|
|
1122
|
+
function rebuildEventStoreIndexLocked(db) {
|
|
1123
|
+
const segmentPaths = listSegmentFiles(db);
|
|
1124
|
+
let indexedEvents = 0;
|
|
1125
|
+
let skippedEvents = 0;
|
|
1126
|
+
const errors = [];
|
|
1127
|
+
db.transaction(() => {
|
|
1128
|
+
db.run("DELETE FROM event_records");
|
|
1129
|
+
db.run("DELETE FROM event_segments");
|
|
1130
|
+
clearCompatibilityProjections(db);
|
|
1131
|
+
for (const path of segmentPaths) {
|
|
1132
|
+
const scan = scanSegmentEvents(db, path, { strict: false });
|
|
1133
|
+
errors.push(...scan.errors);
|
|
1134
|
+
const candidate = segmentCandidate(path);
|
|
1135
|
+
const first = scan.events[0]?.event.event_time ?? null;
|
|
1136
|
+
const last = scan.events.at(-1)?.event.event_time ?? null;
|
|
1137
|
+
db.prepare(`
|
|
1138
|
+
INSERT INTO event_segments (
|
|
1139
|
+
id, relative_path, manifest_path, byte_length, event_count,
|
|
1140
|
+
first_event_time, last_event_time, segment_hash, updated_at
|
|
1141
|
+
)
|
|
1142
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
1143
|
+
`).run(candidate.id, candidate.relative_path, candidate.manifest_path, scan.byte_length, scan.events.length, first, last, scan.segment_hash);
|
|
1144
|
+
for (const item of scan.events) {
|
|
1145
|
+
try {
|
|
1146
|
+
const index = eventIndexFromEnvelope(db, item.event);
|
|
1147
|
+
if (item.event.type === "log")
|
|
1148
|
+
replayLogProjection(db, item.event, index);
|
|
1149
|
+
indexRawEvent(db, index, item.write);
|
|
1150
|
+
applyReplayedCompatibilityProjections(db, item.event, index);
|
|
1151
|
+
indexedEvents += 1;
|
|
1152
|
+
} catch (error) {
|
|
1153
|
+
skippedEvents += 1;
|
|
1154
|
+
errors.push(`Skipped raw event ${item.event.event_id} from ${path}:${item.write.byte_offset}: ${errorMessage(error)}`);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
writeSegmentManifest(db, candidate.id);
|
|
1158
|
+
}
|
|
1159
|
+
})();
|
|
1160
|
+
return {
|
|
1161
|
+
indexed_events: indexedEvents,
|
|
1162
|
+
indexed_segments: segmentPaths.length,
|
|
1163
|
+
skipped_events: skippedEvents,
|
|
1164
|
+
errors
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
function repairEventStoreSegments(db, options = {}) {
|
|
1168
|
+
const apply = options.apply === true;
|
|
1169
|
+
if (apply)
|
|
1170
|
+
return withEventStoreLock(db, () => repairEventStoreSegmentsLocked(db, true));
|
|
1171
|
+
return repairEventStoreSegmentsLocked(db, false);
|
|
1172
|
+
}
|
|
1173
|
+
function repairEventStoreSegmentsLocked(db, apply) {
|
|
1174
|
+
const segmentPaths = listSegmentFiles(db);
|
|
1175
|
+
const repairs = [];
|
|
1176
|
+
const errors = [];
|
|
1177
|
+
for (const segmentPath of segmentPaths) {
|
|
1178
|
+
try {
|
|
1179
|
+
const plan = planSegmentRepair(db, segmentPath);
|
|
1180
|
+
if (plan.repair.removed_lines.length === 0)
|
|
1181
|
+
continue;
|
|
1182
|
+
if (apply)
|
|
1183
|
+
applySegmentRepair(db, plan);
|
|
1184
|
+
repairs.push({ ...plan.repair, applied: apply });
|
|
1185
|
+
} catch (error) {
|
|
1186
|
+
errors.push(`Failed to repair ${segmentPath}: ${errorMessage(error)}`);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
let rebuild;
|
|
1190
|
+
let verification;
|
|
1191
|
+
if (apply) {
|
|
1192
|
+
rebuild = rebuildEventStoreIndex(db);
|
|
1193
|
+
verification = verifyEventStore(db);
|
|
1194
|
+
}
|
|
1195
|
+
return {
|
|
1196
|
+
applied: apply,
|
|
1197
|
+
scanned_segments: segmentPaths.length,
|
|
1198
|
+
repaired_segments: repairs.length,
|
|
1199
|
+
quarantined_bytes: repairs.reduce((total, repair) => total + repair.removed_bytes, 0),
|
|
1200
|
+
repairs,
|
|
1201
|
+
rebuild,
|
|
1202
|
+
verification,
|
|
1203
|
+
errors
|
|
1204
|
+
};
|
|
1205
|
+
}
|
|
1206
|
+
function getActiveSegment(db, ingestTime, eventByteLength) {
|
|
1207
|
+
const prefix = segmentPrefix(ingestTime);
|
|
1208
|
+
const maxSegmentBytes = getMaxSegmentBytes();
|
|
1209
|
+
const latest = db.prepare(`
|
|
1210
|
+
SELECT id, relative_path, byte_length, sealed_at
|
|
1211
|
+
FROM event_segments
|
|
1212
|
+
WHERE relative_path LIKE ?
|
|
1213
|
+
ORDER BY relative_path DESC
|
|
1214
|
+
LIMIT 1
|
|
1215
|
+
`).get(`${prefix}/events-%.jsonl`);
|
|
1216
|
+
if (latest && !latest.sealed_at && latest.byte_length + eventByteLength <= maxSegmentBytes) {
|
|
1217
|
+
return segmentCandidate(latest.relative_path);
|
|
1218
|
+
}
|
|
1219
|
+
if (latest && !latest.sealed_at) {
|
|
1220
|
+
sealSegment(db, latest.id);
|
|
1221
|
+
}
|
|
1222
|
+
const lastSequence = latest ? parseSegmentSequence(latest.relative_path) : 0;
|
|
1223
|
+
return segmentCandidate(`${prefix}/events-${String(lastSequence + 1).padStart(6, "0")}.jsonl`);
|
|
1224
|
+
}
|
|
1225
|
+
function segmentPrefix(isoTime) {
|
|
1226
|
+
const date = new Date(isoTime);
|
|
1227
|
+
const valid = Number.isNaN(date.getTime()) ? new Date : date;
|
|
1228
|
+
const year = valid.getUTCFullYear();
|
|
1229
|
+
const month = String(valid.getUTCMonth() + 1).padStart(2, "0");
|
|
1230
|
+
const day = String(valid.getUTCDate()).padStart(2, "0");
|
|
1231
|
+
return `segments/${year}/${month}/${day}`;
|
|
1232
|
+
}
|
|
1233
|
+
function segmentCandidate(relativePath) {
|
|
1234
|
+
return {
|
|
1235
|
+
id: sha256(relativePath).slice(0, 24),
|
|
1236
|
+
relative_path: relativePath,
|
|
1237
|
+
manifest_path: relativePath.replace(/\.jsonl$/, ".manifest.json")
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
function parseSegmentSequence(relativePath) {
|
|
1241
|
+
const match = /events-(\d+)\.jsonl$/.exec(relativePath);
|
|
1242
|
+
return match?.[1] ? Number.parseInt(match[1], 10) : 0;
|
|
1243
|
+
}
|
|
1244
|
+
function writeSegmentManifest(db, segmentId) {
|
|
1245
|
+
const segment = db.prepare("SELECT * FROM event_segments WHERE id = ?").get(segmentId);
|
|
1246
|
+
if (!segment || typeof segment.manifest_path !== "string")
|
|
1247
|
+
return;
|
|
1248
|
+
const manifestPath = resolveDataPath(db, segment.manifest_path);
|
|
1249
|
+
mkdirSync(dirname(manifestPath), { recursive: true });
|
|
1250
|
+
writeFileSync(manifestPath, `${JSON.stringify({ schema_version: 1, segment }, null, 2)}
|
|
1251
|
+
`, "utf8");
|
|
1252
|
+
}
|
|
1253
|
+
function scanSegmentEvents(db, relativePath, options) {
|
|
1254
|
+
const absolutePath = resolveDataPath(db, relativePath);
|
|
1255
|
+
const bytes = readFileSync(absolutePath);
|
|
1256
|
+
const candidate = segmentCandidate(relativePath);
|
|
1257
|
+
const events = [];
|
|
1258
|
+
const errors = [];
|
|
1259
|
+
let offset = 0;
|
|
1260
|
+
while (offset < bytes.byteLength) {
|
|
1261
|
+
const newline = bytes.indexOf(10, offset);
|
|
1262
|
+
if (newline === -1) {
|
|
1263
|
+
const message = `Partial raw event line in ${relativePath} at byte ${offset}`;
|
|
1264
|
+
if (options.strict)
|
|
1265
|
+
throw new Error(message);
|
|
1266
|
+
errors.push(message);
|
|
1267
|
+
break;
|
|
1268
|
+
}
|
|
1269
|
+
const end = newline + 1;
|
|
1270
|
+
const line = bytes.subarray(offset, end);
|
|
1271
|
+
if (line.toString("utf8").trim().length > 0) {
|
|
1272
|
+
try {
|
|
1273
|
+
const event = JSON.parse(line.toString("utf8"));
|
|
1274
|
+
events.push({
|
|
1275
|
+
event,
|
|
1276
|
+
write: {
|
|
1277
|
+
segment_id: candidate.id,
|
|
1278
|
+
segment_path: candidate.relative_path,
|
|
1279
|
+
manifest_path: candidate.manifest_path,
|
|
1280
|
+
byte_offset: offset,
|
|
1281
|
+
byte_length: line.byteLength,
|
|
1282
|
+
record_hash: sha256(line)
|
|
1283
|
+
}
|
|
1284
|
+
});
|
|
1285
|
+
} catch (error) {
|
|
1286
|
+
const message = `Malformed raw event line in ${relativePath} at byte ${offset}: ${errorMessage(error)}`;
|
|
1287
|
+
if (options.strict)
|
|
1288
|
+
throw new Error(message);
|
|
1289
|
+
errors.push(message);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
offset = end;
|
|
1293
|
+
}
|
|
1294
|
+
return {
|
|
1295
|
+
events,
|
|
1296
|
+
errors,
|
|
1297
|
+
byte_length: bytes.byteLength,
|
|
1298
|
+
segment_hash: sha256(bytes)
|
|
1299
|
+
};
|
|
1300
|
+
}
|
|
1301
|
+
function planSegmentRepair(db, relativePath) {
|
|
1302
|
+
const absolutePath = resolveDataPath(db, relativePath);
|
|
1303
|
+
const bytes = readFileSync(absolutePath);
|
|
1304
|
+
const originalHash = sha256(bytes);
|
|
1305
|
+
const retainedChunks = [];
|
|
1306
|
+
const quarantineChunks = [];
|
|
1307
|
+
const removedLines = [];
|
|
1308
|
+
let retainedEvents = 0;
|
|
1309
|
+
let malformedLines = 0;
|
|
1310
|
+
let partialTruncated = false;
|
|
1311
|
+
let offset = 0;
|
|
1312
|
+
while (offset < bytes.byteLength) {
|
|
1313
|
+
const newline = bytes.indexOf(10, offset);
|
|
1314
|
+
if (newline === -1) {
|
|
1315
|
+
const line2 = bytes.subarray(offset);
|
|
1316
|
+
const message = `Partial raw event line in ${relativePath} at byte ${offset}`;
|
|
1317
|
+
quarantineChunks.push(line2);
|
|
1318
|
+
removedLines.push({
|
|
1319
|
+
byte_offset: offset,
|
|
1320
|
+
byte_length: line2.byteLength,
|
|
1321
|
+
reason: "partial",
|
|
1322
|
+
message
|
|
1323
|
+
});
|
|
1324
|
+
partialTruncated = true;
|
|
1325
|
+
break;
|
|
1326
|
+
}
|
|
1327
|
+
const end = newline + 1;
|
|
1328
|
+
const line = bytes.subarray(offset, end);
|
|
1329
|
+
if (line.toString("utf8").trim().length === 0) {
|
|
1330
|
+
retainedChunks.push(line);
|
|
1331
|
+
offset = end;
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
try {
|
|
1335
|
+
JSON.parse(line.toString("utf8"));
|
|
1336
|
+
retainedChunks.push(line);
|
|
1337
|
+
retainedEvents += 1;
|
|
1338
|
+
} catch (error) {
|
|
1339
|
+
const message = `Malformed raw event line in ${relativePath} at byte ${offset}: ${errorMessage(error)}`;
|
|
1340
|
+
quarantineChunks.push(line);
|
|
1341
|
+
removedLines.push({
|
|
1342
|
+
byte_offset: offset,
|
|
1343
|
+
byte_length: line.byteLength,
|
|
1344
|
+
reason: "malformed",
|
|
1345
|
+
message
|
|
1346
|
+
});
|
|
1347
|
+
malformedLines += 1;
|
|
1348
|
+
}
|
|
1349
|
+
offset = end;
|
|
1350
|
+
}
|
|
1351
|
+
const repairedBytes = Buffer.concat(retainedChunks);
|
|
1352
|
+
const quarantineBytes = Buffer.concat(quarantineChunks);
|
|
1353
|
+
const repairedHash = sha256(repairedBytes);
|
|
1354
|
+
const quarantinePath = quarantinePathForSegment(relativePath, originalHash);
|
|
1355
|
+
const quarantineManifestPath = quarantinePath.replace(/\.bad$/, ".manifest.json");
|
|
1356
|
+
return {
|
|
1357
|
+
repaired_bytes: repairedBytes,
|
|
1358
|
+
quarantine_bytes: quarantineBytes,
|
|
1359
|
+
repair: {
|
|
1360
|
+
segment_path: relativePath,
|
|
1361
|
+
quarantine_path: quarantinePath,
|
|
1362
|
+
quarantine_manifest_path: quarantineManifestPath,
|
|
1363
|
+
original_byte_length: bytes.byteLength,
|
|
1364
|
+
repaired_byte_length: repairedBytes.byteLength,
|
|
1365
|
+
original_hash: originalHash,
|
|
1366
|
+
repaired_hash: repairedHash,
|
|
1367
|
+
retained_events: retainedEvents,
|
|
1368
|
+
removed_lines: removedLines,
|
|
1369
|
+
removed_bytes: quarantineBytes.byteLength,
|
|
1370
|
+
partial_truncated: partialTruncated,
|
|
1371
|
+
malformed_lines: malformedLines,
|
|
1372
|
+
applied: false
|
|
1373
|
+
}
|
|
1374
|
+
};
|
|
1375
|
+
}
|
|
1376
|
+
function applySegmentRepair(db, plan) {
|
|
1377
|
+
const repair = plan.repair;
|
|
1378
|
+
const segmentPath = resolveDataPath(db, repair.segment_path);
|
|
1379
|
+
const quarantinePath = resolveDataPath(db, repair.quarantine_path);
|
|
1380
|
+
const quarantineManifestPath = resolveDataPath(db, repair.quarantine_manifest_path);
|
|
1381
|
+
mkdirSync(dirname(quarantinePath), { recursive: true });
|
|
1382
|
+
mkdirSync(dirname(quarantineManifestPath), { recursive: true });
|
|
1383
|
+
writeFileSync(quarantinePath, plan.quarantine_bytes);
|
|
1384
|
+
writeFileSync(quarantineManifestPath, `${JSON.stringify({
|
|
1385
|
+
schema_version: 1,
|
|
1386
|
+
original_segment_path: repair.segment_path,
|
|
1387
|
+
quarantine_path: repair.quarantine_path,
|
|
1388
|
+
original_byte_length: repair.original_byte_length,
|
|
1389
|
+
repaired_byte_length: repair.repaired_byte_length,
|
|
1390
|
+
original_hash: repair.original_hash,
|
|
1391
|
+
repaired_hash: repair.repaired_hash,
|
|
1392
|
+
removed_bytes: repair.removed_bytes,
|
|
1393
|
+
removed_lines: repair.removed_lines,
|
|
1394
|
+
repaired_at: new Date().toISOString()
|
|
1395
|
+
}, null, 2)}
|
|
1396
|
+
`, "utf8");
|
|
1397
|
+
const tmpPath = `${segmentPath}.repair-${process.pid}-${Date.now()}.tmp`;
|
|
1398
|
+
writeFileSync(tmpPath, plan.repaired_bytes);
|
|
1399
|
+
renameSync(tmpPath, segmentPath);
|
|
1400
|
+
}
|
|
1401
|
+
function quarantinePathForSegment(relativePath, originalHash) {
|
|
1402
|
+
return `quarantine/${relativePath.replace(/\.jsonl$/, `.${originalHash.slice(0, 12)}.bad`)}`;
|
|
1403
|
+
}
|
|
1404
|
+
function listSegmentFiles(db) {
|
|
1405
|
+
const root = resolveDataPath(db, "segments");
|
|
1406
|
+
if (!existsSync(root))
|
|
1407
|
+
return [];
|
|
1408
|
+
const paths = [];
|
|
1409
|
+
walkSegmentDir(root, "segments", paths);
|
|
1410
|
+
return paths.filter((path) => path.endsWith(".jsonl")).sort();
|
|
1411
|
+
}
|
|
1412
|
+
function walkSegmentDir(absoluteDir, relativeDir, paths) {
|
|
1413
|
+
for (const entry of readdirSync(absoluteDir, { withFileTypes: true })) {
|
|
1414
|
+
const relativePath = `${relativeDir}/${entry.name}`;
|
|
1415
|
+
const absolutePath = join(absoluteDir, entry.name);
|
|
1416
|
+
if (entry.isDirectory()) {
|
|
1417
|
+
walkSegmentDir(absolutePath, relativePath, paths);
|
|
1418
|
+
} else if (entry.isFile()) {
|
|
1419
|
+
paths.push(relativePath);
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
function hasSourceMapContainer(record) {
|
|
1424
|
+
if (Object.keys(objectRecord3(record.source_map)).length > 0)
|
|
1425
|
+
return true;
|
|
1426
|
+
const artifact = objectRecord3(record.artifact);
|
|
1427
|
+
if (Object.keys(objectRecord3(artifact.source_map)).length > 0)
|
|
1428
|
+
return true;
|
|
1429
|
+
const artifactType = stringAttribute(record, "artifact_type") ?? stringAttribute(record, "type") ?? stringAttribute(artifact, "artifact_type") ?? stringAttribute(artifact, "type");
|
|
1430
|
+
const path = stringAttribute(record, "path") ?? stringAttribute(artifact, "path");
|
|
1431
|
+
return artifactType === "source_map" || artifactType === "source-map" || artifactType === "sourcemap" || Boolean(path?.endsWith(".map"));
|
|
1432
|
+
}
|
|
1433
|
+
function eventIndexFromEnvelope(db, event) {
|
|
1434
|
+
const rawAttrs = objectRecord3(event.attributes);
|
|
1435
|
+
const rawBody = objectRecord3(event.body);
|
|
1436
|
+
const body = event.type === "artifact" ? sanitizeSourceMapArtifactRecord(rawBody) : rawBody;
|
|
1437
|
+
const artifact = sanitizeSourceMapArtifactRecord(objectRecord3(rawBody.artifact));
|
|
1438
|
+
const attrs = event.type === "artifact" && (hasSourceMapContainer(body) || hasSourceMapContainer(artifact) || hasSourceMapContainer(rawAttrs)) ? sanitizeSourceMapContextRecord(rawAttrs) : rawAttrs;
|
|
1439
|
+
const log = objectRecord3(event.body?.log);
|
|
1440
|
+
const rawProjectId = stringAttribute(attrs, "project_id") ?? stringAttribute(log, "project_id");
|
|
1441
|
+
const rawPageId = stringAttribute(attrs, "page_id") ?? stringAttribute(log, "page_id");
|
|
1442
|
+
return {
|
|
1443
|
+
event_id: event.event_id,
|
|
1444
|
+
schema_version: event.schema_version,
|
|
1445
|
+
source_event_id: event.source_event_id ?? null,
|
|
1446
|
+
event_type: event.type,
|
|
1447
|
+
event_time: event.event_time,
|
|
1448
|
+
ingest_time: event.ingest_time,
|
|
1449
|
+
severity: event.severity ?? null,
|
|
1450
|
+
source: event.source,
|
|
1451
|
+
project_id: rawProjectId && rowExists(db, "projects", rawProjectId) ? rawProjectId : null,
|
|
1452
|
+
page_id: rawPageId && rowExists(db, "pages", rawPageId) ? rawPageId : null,
|
|
1453
|
+
log_id: event.type === "log" ? event.event_id : null,
|
|
1454
|
+
machine_id: event.machine_id ?? stringAttribute(attrs, "machine_id"),
|
|
1455
|
+
repo_id: event.repo_id ?? stringAttribute(attrs, "repo_id"),
|
|
1456
|
+
app_id: event.app_id ?? stringAttribute(attrs, "app_id"),
|
|
1457
|
+
process_id: event.process_id ?? stringAttribute(attrs, "process_id"),
|
|
1458
|
+
run_id: event.run_id ?? stringAttribute(attrs, "run_id"),
|
|
1459
|
+
trace_id: event.trace_id ?? stringAttribute(attrs, "trace_id") ?? stringAttribute(log, "trace_id"),
|
|
1460
|
+
span_id: event.span_id ?? stringAttribute(attrs, "span_id"),
|
|
1461
|
+
parent_span_id: event.parent_span_id ?? stringAttribute(attrs, "parent_span_id"),
|
|
1462
|
+
session_id: event.session_id ?? stringAttribute(attrs, "session_id") ?? stringAttribute(log, "session_id"),
|
|
1463
|
+
release_id: event.release_id ?? stringAttribute(attrs, "release_id"),
|
|
1464
|
+
environment: event.environment ?? stringAttribute(attrs, "environment"),
|
|
1465
|
+
artifact_id: stringAttribute(attrs, "artifact_id") ?? stringAttribute(body, "artifact_id") ?? stringAttribute(artifact, "artifact_id"),
|
|
1466
|
+
privacy_tier: event.privacy ?? stringAttribute(attrs, "privacy_tier"),
|
|
1467
|
+
message: event.message ?? null,
|
|
1468
|
+
metadata: metadataFromEnvelope(event)
|
|
1469
|
+
};
|
|
1470
|
+
}
|
|
1471
|
+
function clearCompatibilityProjections(db) {
|
|
1472
|
+
db.run("DELETE FROM issues");
|
|
1473
|
+
db.run("DELETE FROM logs");
|
|
1474
|
+
db.run("DELETE FROM spans");
|
|
1475
|
+
db.run("DELETE FROM traces");
|
|
1476
|
+
db.run("DELETE FROM sessions");
|
|
1477
|
+
db.run("DELETE FROM source_map_sources");
|
|
1478
|
+
db.run("DELETE FROM source_maps");
|
|
1479
|
+
db.run("DELETE FROM test_cases");
|
|
1480
|
+
db.run("DELETE FROM test_reports");
|
|
1481
|
+
db.run("DELETE FROM artifacts");
|
|
1482
|
+
db.run("DELETE FROM releases");
|
|
1483
|
+
db.run("DELETE FROM processes");
|
|
1484
|
+
db.run("DELETE FROM runs");
|
|
1485
|
+
}
|
|
1486
|
+
function applyReplayedCompatibilityProjections(db, event, index) {
|
|
1487
|
+
const attrs = objectRecord3(event.attributes);
|
|
1488
|
+
const category = stringAttribute(attrs, "category");
|
|
1489
|
+
const isCommandRunSummary = event.type === "build" && category === "command_run_summary";
|
|
1490
|
+
const isTestReportEvent = event.type === "build" && category === "test_report";
|
|
1491
|
+
if (index.trace_id)
|
|
1492
|
+
replayTraceProjection(db, event, index);
|
|
1493
|
+
if (event.type === "span")
|
|
1494
|
+
replaySpanProjection(db, event, index);
|
|
1495
|
+
if (index.session_id || event.type === "session")
|
|
1496
|
+
replaySessionProjection(db, event, index);
|
|
1497
|
+
if (index.release_id || event.type === "release")
|
|
1498
|
+
replayReleaseProjection(db, event, index);
|
|
1499
|
+
if (index.artifact_id || event.type === "artifact") {
|
|
1500
|
+
replayArtifactProjection(db, event, index);
|
|
1501
|
+
upsertSourceMapProjection(db, event, index);
|
|
1502
|
+
}
|
|
1503
|
+
if (isTestReportEvent)
|
|
1504
|
+
upsertTestReportProjection(db, event, index);
|
|
1505
|
+
if (isCommandRunSummary) {
|
|
1506
|
+
replayProcessRunProjection(db, event, index, {
|
|
1507
|
+
preserveExistingMetadata: true
|
|
1508
|
+
});
|
|
1509
|
+
} else if (!isTestReportEvent && (index.process_id || index.run_id || event.type === "process" || event.type === "build")) {
|
|
1510
|
+
replayProcessRunProjection(db, event, index);
|
|
1511
|
+
}
|
|
1512
|
+
replayIssueProjection(db, event, index);
|
|
1513
|
+
}
|
|
1514
|
+
function replayLogProjection(db, event, index) {
|
|
1515
|
+
const log = objectRecord3(event.body?.log);
|
|
1516
|
+
const level = normalizeLogLevel(event.severity ?? stringAttribute(log, "level"));
|
|
1517
|
+
const metadata = objectRecord3(log.metadata);
|
|
1518
|
+
db.prepare(`
|
|
1519
|
+
INSERT INTO logs (id, timestamp, project_id, page_id, level, source, service, message, trace_id, session_id, agent, url, stack_trace, metadata)
|
|
1520
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1521
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1522
|
+
timestamp = excluded.timestamp,
|
|
1523
|
+
project_id = excluded.project_id,
|
|
1524
|
+
page_id = excluded.page_id,
|
|
1525
|
+
level = excluded.level,
|
|
1526
|
+
source = excluded.source,
|
|
1527
|
+
service = excluded.service,
|
|
1528
|
+
message = excluded.message,
|
|
1529
|
+
trace_id = excluded.trace_id,
|
|
1530
|
+
session_id = excluded.session_id,
|
|
1531
|
+
agent = excluded.agent,
|
|
1532
|
+
url = excluded.url,
|
|
1533
|
+
stack_trace = excluded.stack_trace,
|
|
1534
|
+
metadata = excluded.metadata
|
|
1535
|
+
`).run(event.event_id, event.event_time, index.project_id ?? null, index.page_id ?? null, level, stringAttribute(log, "source") ?? event.source, stringAttribute(log, "service") ?? stringAttribute(objectRecord3(event.attributes), "service"), stringAttribute(log, "message") ?? event.message ?? "", index.trace_id ?? null, index.session_id ?? null, stringAttribute(log, "agent"), stringAttribute(log, "url"), stringAttribute(log, "stack_trace"), Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null);
|
|
1536
|
+
}
|
|
1537
|
+
function replayTraceProjection(db, event, index) {
|
|
1538
|
+
if (!index.trace_id)
|
|
1539
|
+
return;
|
|
1540
|
+
const attrs = objectRecord3(event.attributes);
|
|
1541
|
+
const body = objectRecord3(event.body);
|
|
1542
|
+
db.prepare(`
|
|
1543
|
+
INSERT INTO traces (id, project_id, app_id, root_span_id, started_at, ended_at, status, metadata)
|
|
1544
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1545
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1546
|
+
project_id = COALESCE(traces.project_id, excluded.project_id),
|
|
1547
|
+
app_id = COALESCE(traces.app_id, excluded.app_id),
|
|
1548
|
+
root_span_id = COALESCE(traces.root_span_id, excluded.root_span_id),
|
|
1549
|
+
started_at = CASE
|
|
1550
|
+
WHEN excluded.started_at IS NOT NULL
|
|
1551
|
+
AND (traces.started_at IS NULL OR excluded.started_at < traces.started_at)
|
|
1552
|
+
THEN excluded.started_at
|
|
1553
|
+
ELSE traces.started_at
|
|
1554
|
+
END,
|
|
1555
|
+
ended_at = COALESCE(excluded.ended_at, traces.ended_at),
|
|
1556
|
+
status = COALESCE(excluded.status, traces.status),
|
|
1557
|
+
metadata = excluded.metadata
|
|
1558
|
+
`).run(index.trace_id, index.project_id ?? null, index.app_id ?? null, index.span_id ?? null, stringAttribute(attrs, "started_at") ?? event.event_time, stringAttribute(attrs, "ended_at") ?? stringAttribute(body, "ended_at"), stringAttribute(attrs, "status") ?? stringAttribute(body, "status"), JSON.stringify(index.metadata ?? {}));
|
|
1559
|
+
}
|
|
1560
|
+
function replaySpanProjection(db, event, index) {
|
|
1561
|
+
const attrs = objectRecord3(event.attributes);
|
|
1562
|
+
const body = objectRecord3(event.body);
|
|
1563
|
+
const spanId = index.span_id ?? event.event_id;
|
|
1564
|
+
db.prepare(`
|
|
1565
|
+
INSERT INTO spans (id, trace_id, parent_span_id, app_id, process_id, name, operation, status, started_at, ended_at, duration_ms, metadata)
|
|
1566
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1567
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1568
|
+
trace_id = COALESCE(spans.trace_id, excluded.trace_id),
|
|
1569
|
+
parent_span_id = COALESCE(spans.parent_span_id, excluded.parent_span_id),
|
|
1570
|
+
app_id = COALESCE(spans.app_id, excluded.app_id),
|
|
1571
|
+
process_id = COALESCE(spans.process_id, excluded.process_id),
|
|
1572
|
+
name = COALESCE(spans.name, excluded.name),
|
|
1573
|
+
operation = COALESCE(spans.operation, excluded.operation),
|
|
1574
|
+
ended_at = COALESCE(excluded.ended_at, spans.ended_at),
|
|
1575
|
+
duration_ms = COALESCE(excluded.duration_ms, spans.duration_ms),
|
|
1576
|
+
status = COALESCE(excluded.status, spans.status),
|
|
1577
|
+
metadata = excluded.metadata
|
|
1578
|
+
`).run(spanId, index.trace_id ?? null, index.parent_span_id ?? null, index.app_id ?? null, index.process_id ?? null, stringAttribute(attrs, "name") ?? stringAttribute(body, "name") ?? event.message ?? null, stringAttribute(attrs, "operation") ?? stringAttribute(body, "operation"), stringAttribute(attrs, "status") ?? event.severity ?? null, stringAttribute(attrs, "started_at") ?? event.event_time, stringAttribute(attrs, "ended_at"), numberAttribute(attrs, "duration_ms") ?? numberAttribute(body, "duration_ms"), JSON.stringify(index.metadata ?? {}));
|
|
1579
|
+
}
|
|
1580
|
+
function replaySessionProjection(db, event, index) {
|
|
1581
|
+
const attrs = objectRecord3(event.attributes);
|
|
1582
|
+
const sessionId = index.session_id ?? event.event_id;
|
|
1583
|
+
db.prepare(`
|
|
1584
|
+
INSERT INTO sessions (id, project_id, app_id, user_hash, started_at, ended_at, status, metadata)
|
|
1585
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1586
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1587
|
+
project_id = COALESCE(sessions.project_id, excluded.project_id),
|
|
1588
|
+
app_id = COALESCE(sessions.app_id, excluded.app_id),
|
|
1589
|
+
user_hash = COALESCE(sessions.user_hash, excluded.user_hash),
|
|
1590
|
+
started_at = CASE
|
|
1591
|
+
WHEN excluded.started_at IS NOT NULL
|
|
1592
|
+
AND (sessions.started_at IS NULL OR excluded.started_at < sessions.started_at)
|
|
1593
|
+
THEN excluded.started_at
|
|
1594
|
+
ELSE sessions.started_at
|
|
1595
|
+
END,
|
|
1596
|
+
ended_at = COALESCE(excluded.ended_at, sessions.ended_at),
|
|
1597
|
+
status = COALESCE(excluded.status, sessions.status),
|
|
1598
|
+
metadata = excluded.metadata
|
|
1599
|
+
`).run(sessionId, index.project_id ?? null, index.app_id ?? null, stringAttribute(attrs, "user_hash"), stringAttribute(attrs, "started_at") ?? event.event_time, stringAttribute(attrs, "ended_at"), stringAttribute(attrs, "status"), JSON.stringify(index.metadata ?? {}));
|
|
1600
|
+
}
|
|
1601
|
+
function replayReleaseProjection(db, event, index) {
|
|
1602
|
+
const attrs = objectRecord3(event.attributes);
|
|
1603
|
+
const releaseId = index.release_id ?? event.event_id;
|
|
1604
|
+
db.prepare(`
|
|
1605
|
+
INSERT INTO releases (id, project_id, app_id, version, commit_sha, build_id, deployed_at, metadata)
|
|
1606
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1607
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1608
|
+
project_id = COALESCE(releases.project_id, excluded.project_id),
|
|
1609
|
+
app_id = COALESCE(releases.app_id, excluded.app_id),
|
|
1610
|
+
version = COALESCE(releases.version, excluded.version),
|
|
1611
|
+
commit_sha = COALESCE(releases.commit_sha, excluded.commit_sha),
|
|
1612
|
+
build_id = COALESCE(releases.build_id, excluded.build_id),
|
|
1613
|
+
deployed_at = COALESCE(excluded.deployed_at, releases.deployed_at),
|
|
1614
|
+
metadata = excluded.metadata
|
|
1615
|
+
`).run(releaseId, index.project_id ?? null, index.app_id ?? null, stringAttribute(attrs, "version") ?? event.message ?? null, stringAttribute(attrs, "commit_sha"), stringAttribute(attrs, "build_id"), stringAttribute(attrs, "deployed_at") ?? event.event_time, JSON.stringify(index.metadata ?? {}));
|
|
1616
|
+
}
|
|
1617
|
+
function replayArtifactProjection(db, event, index) {
|
|
1618
|
+
const sanitizedAttrs = sanitizeSourceMapArtifactRecord(objectRecord3(event.attributes));
|
|
1619
|
+
const rawBody = objectRecord3(event.body);
|
|
1620
|
+
const body = sanitizeSourceMapArtifactRecord(rawBody);
|
|
1621
|
+
const artifact = sanitizeSourceMapArtifactRecord(objectRecord3(rawBody.artifact));
|
|
1622
|
+
const attrs = hasSourceMapContainer(body) || hasSourceMapContainer(artifact) || hasSourceMapContainer(sanitizedAttrs) ? sanitizeSourceMapContextRecord(sanitizedAttrs) : sanitizedAttrs;
|
|
1623
|
+
const isSourceMapArtifact = hasSourceMapContainer(body) || hasSourceMapContainer(artifact) || hasSourceMapContainer(attrs);
|
|
1624
|
+
const artifactIdCandidate = index.artifact_id ?? stringAttribute(attrs, "artifact_id") ?? stringAttribute(body, "artifact_id") ?? stringAttribute(artifact, "artifact_id");
|
|
1625
|
+
const artifactId = isSourceMapArtifact ? sanitizeSourceMapIdentifierValue(artifactIdCandidate) ?? sourceMapFallbackIdentifier(event.event_id) : artifactIdCandidate ?? event.event_id;
|
|
1626
|
+
db.prepare(`
|
|
1627
|
+
INSERT INTO artifacts (id, release_id, artifact_type, path, content_hash, size_bytes, metadata)
|
|
1628
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1629
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1630
|
+
release_id = COALESCE(artifacts.release_id, excluded.release_id),
|
|
1631
|
+
artifact_type = COALESCE(artifacts.artifact_type, excluded.artifact_type),
|
|
1632
|
+
path = COALESCE(excluded.path, artifacts.path),
|
|
1633
|
+
content_hash = COALESCE(excluded.content_hash, artifacts.content_hash),
|
|
1634
|
+
size_bytes = COALESCE(excluded.size_bytes, artifacts.size_bytes),
|
|
1635
|
+
metadata = excluded.metadata
|
|
1636
|
+
`).run(artifactId, index.release_id ?? null, stringAttribute(attrs, "artifact_type") ?? stringAttribute(attrs, "type") ?? stringAttribute(artifact, "artifact_type") ?? stringAttribute(artifact, "type") ?? event.type, stringAttribute(attrs, "path") ?? stringAttribute(body, "path") ?? stringAttribute(artifact, "path"), stringAttribute(attrs, "content_hash") ?? stringAttribute(body, "content_hash") ?? stringAttribute(artifact, "content_hash"), numberAttribute(attrs, "size_bytes") ?? numberAttribute(body, "size_bytes") ?? numberAttribute(artifact, "size_bytes"), JSON.stringify(index.metadata ?? {}));
|
|
1637
|
+
}
|
|
1638
|
+
function replayProcessRunProjection(db, event, index, options = {}) {
|
|
1639
|
+
const attrs = objectRecord3(event.attributes);
|
|
1640
|
+
const processBody = objectRecord3(event.body?.process);
|
|
1641
|
+
const lifecycleBody = objectRecord3(event.body?.lifecycle);
|
|
1642
|
+
const body = Object.keys(processBody).length > 0 ? processBody : lifecycleBody;
|
|
1643
|
+
const preserveExistingMetadata = options.preserveExistingMetadata ? 1 : 0;
|
|
1644
|
+
const metadataObject = index.metadata ?? {};
|
|
1645
|
+
const metadata = JSON.stringify(metadataObject);
|
|
1646
|
+
const processMetadata = preserveExistingMetadata && index.process_id ? mergedReplayMetadata(db, "processes", index.process_id, metadataObject) : metadata;
|
|
1647
|
+
const runMetadata = preserveExistingMetadata && index.run_id ? mergedReplayMetadata(db, "runs", index.run_id, metadataObject) : metadata;
|
|
1648
|
+
const exitCode = numberAttribute(attrs, "exit_code") ?? numberAttribute(body, "exit_code");
|
|
1649
|
+
const signal = stringAttribute(attrs, "signal") ?? stringAttribute(body, "signal");
|
|
1650
|
+
const status = stringAttribute(attrs, "status") ?? stringAttribute(body, "status") ?? statusFromExit(exitCode, signal);
|
|
1651
|
+
if (index.process_id) {
|
|
1652
|
+
db.prepare(`
|
|
1653
|
+
INSERT INTO processes (id, machine_id, repo_id, app_id, pid, ppid, command, cwd, started_at, ended_at, exit_code, metadata)
|
|
1654
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1655
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1656
|
+
machine_id = COALESCE(processes.machine_id, excluded.machine_id),
|
|
1657
|
+
repo_id = COALESCE(processes.repo_id, excluded.repo_id),
|
|
1658
|
+
app_id = COALESCE(processes.app_id, excluded.app_id),
|
|
1659
|
+
pid = COALESCE(processes.pid, excluded.pid),
|
|
1660
|
+
ppid = COALESCE(processes.ppid, excluded.ppid),
|
|
1661
|
+
command = COALESCE(processes.command, excluded.command),
|
|
1662
|
+
cwd = COALESCE(processes.cwd, excluded.cwd),
|
|
1663
|
+
ended_at = COALESCE(excluded.ended_at, processes.ended_at),
|
|
1664
|
+
exit_code = COALESCE(excluded.exit_code, processes.exit_code),
|
|
1665
|
+
metadata = excluded.metadata
|
|
1666
|
+
`).run(index.process_id, index.machine_id ?? null, index.repo_id ?? null, index.app_id ?? null, numberAttribute(attrs, "pid") ?? numberAttribute(body, "pid"), numberAttribute(attrs, "ppid") ?? numberAttribute(body, "ppid"), commandString(stringAttribute(attrs, "command") ?? body.command), stringAttribute(attrs, "cwd") ?? stringAttribute(body, "cwd"), stringAttribute(attrs, "started_at") ?? stringAttribute(body, "started_at") ?? event.event_time, stringAttribute(attrs, "ended_at") ?? stringAttribute(body, "ended_at"), exitCode, processMetadata);
|
|
1667
|
+
}
|
|
1668
|
+
if (index.run_id) {
|
|
1669
|
+
db.prepare(`
|
|
1670
|
+
INSERT INTO runs (id, process_id, run_type, name, status, started_at, ended_at, exit_code, metadata)
|
|
1671
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1672
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1673
|
+
process_id = COALESCE(runs.process_id, excluded.process_id),
|
|
1674
|
+
run_type = COALESCE(runs.run_type, excluded.run_type),
|
|
1675
|
+
name = COALESCE(runs.name, excluded.name),
|
|
1676
|
+
ended_at = COALESCE(excluded.ended_at, runs.ended_at),
|
|
1677
|
+
exit_code = COALESCE(excluded.exit_code, runs.exit_code),
|
|
1678
|
+
status = COALESCE(excluded.status, runs.status),
|
|
1679
|
+
metadata = excluded.metadata
|
|
1680
|
+
`).run(index.run_id, index.process_id ?? null, stringAttribute(attrs, "run_type") ?? stringAttribute(body, "run_type") ?? stringAttribute(body, "kind") ?? event.type, stringAttribute(attrs, "name") ?? commandString(body.command) ?? event.message ?? null, status, stringAttribute(attrs, "started_at") ?? stringAttribute(body, "started_at") ?? event.event_time, stringAttribute(attrs, "ended_at") ?? stringAttribute(body, "ended_at"), exitCode, runMetadata);
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
function mergedReplayMetadata(db, table, id, incoming) {
|
|
1684
|
+
const row = db.prepare(`SELECT metadata FROM ${table} WHERE id = ?`).get(id);
|
|
1685
|
+
const existing = parseJsonRecord(row?.metadata);
|
|
1686
|
+
return JSON.stringify({ ...incoming, ...existing });
|
|
1687
|
+
}
|
|
1688
|
+
function statusFromExit(exitCode, signal) {
|
|
1689
|
+
if (signal)
|
|
1690
|
+
return "failed";
|
|
1691
|
+
if (exitCode === null)
|
|
1692
|
+
return null;
|
|
1693
|
+
return exitCode === 0 ? "completed" : "failed";
|
|
1694
|
+
}
|
|
1695
|
+
function replayIssueProjection(db, event, index) {
|
|
1696
|
+
if (event.type === "exception" && event.message) {
|
|
1697
|
+
upsertIssue(db, {
|
|
1698
|
+
project_id: index.project_id ?? undefined,
|
|
1699
|
+
level: event.severity ?? "error",
|
|
1700
|
+
service: stringAttribute(objectRecord3(event.attributes), "service"),
|
|
1701
|
+
message: event.message,
|
|
1702
|
+
stack_trace: stringAttribute(objectRecord3(event.attributes), "stack_trace") ?? stringAttribute(objectRecord3(event.body), "stack_trace")
|
|
1703
|
+
});
|
|
1704
|
+
return;
|
|
1705
|
+
}
|
|
1706
|
+
if (event.type !== "log" || !index.project_id || !event.message || !index.severity || !["warn", "error", "fatal"].includes(index.severity))
|
|
1707
|
+
return;
|
|
1708
|
+
const log = objectRecord3(event.body?.log);
|
|
1709
|
+
upsertIssue(db, {
|
|
1710
|
+
project_id: index.project_id,
|
|
1711
|
+
level: index.severity,
|
|
1712
|
+
service: stringAttribute(log, "service") ?? stringAttribute(objectRecord3(event.attributes), "service"),
|
|
1713
|
+
message: event.message,
|
|
1714
|
+
stack_trace: stringAttribute(log, "stack_trace")
|
|
1715
|
+
});
|
|
1716
|
+
}
|
|
1717
|
+
function metadataFromEnvelope(event) {
|
|
1718
|
+
const attrs = objectRecord3(event.attributes);
|
|
1719
|
+
const body = objectRecord3(event.body);
|
|
1720
|
+
const log = objectRecord3(body.log);
|
|
1721
|
+
const logMetadata = objectRecord3(log.metadata);
|
|
1722
|
+
if (event.type === "log")
|
|
1723
|
+
return Object.keys(logMetadata).length > 0 ? logMetadata : null;
|
|
1724
|
+
const processBody = objectRecord3(body.process);
|
|
1725
|
+
if (Object.keys(processBody).length > 0)
|
|
1726
|
+
return { ...attrs, ...processBody };
|
|
1727
|
+
const chunkBody = objectRecord3(body.process_stream_chunk);
|
|
1728
|
+
if (Object.keys(chunkBody).length > 0)
|
|
1729
|
+
return { ...attrs, ...chunkBody };
|
|
1730
|
+
const lifecycleBody = objectRecord3(body.lifecycle);
|
|
1731
|
+
if (Object.keys(lifecycleBody).length > 0)
|
|
1732
|
+
return { ...attrs, ...lifecycleBody };
|
|
1733
|
+
const testReportBody = objectRecord3(body.test_report);
|
|
1734
|
+
if (Object.keys(testReportBody).length > 0)
|
|
1735
|
+
return sanitizedTestReportMetadata(testReportBody, attrs, attrs);
|
|
1736
|
+
const artifactBody = objectRecord3(body.artifact);
|
|
1737
|
+
if (Object.keys(artifactBody).length > 0)
|
|
1738
|
+
return sanitizedArtifactMetadata({ ...artifactBody, ...attrs });
|
|
1739
|
+
if (event.type === "artifact" && Object.keys(body).length > 0)
|
|
1740
|
+
return sanitizedArtifactMetadata({ ...body, ...attrs });
|
|
1741
|
+
if (event.type === "artifact" && Object.keys(attrs).length > 0)
|
|
1742
|
+
return sanitizedArtifactMetadata(attrs);
|
|
1743
|
+
return Object.keys(attrs).length > 0 ? attrs : null;
|
|
1744
|
+
}
|
|
1745
|
+
function sanitizedArtifactMetadata(metadata) {
|
|
1746
|
+
const output = { ...metadata };
|
|
1747
|
+
if ("source_map" in output) {
|
|
1748
|
+
const sourceMap = sanitizeSourceMapTelemetry(output.source_map);
|
|
1749
|
+
if (sourceMap)
|
|
1750
|
+
output.source_map = sourceMap;
|
|
1751
|
+
else
|
|
1752
|
+
output.source_map = undefined;
|
|
1753
|
+
}
|
|
1754
|
+
return sanitizeSourceMapArtifactRecord(output);
|
|
1755
|
+
}
|
|
1756
|
+
function objectRecord3(value) {
|
|
1757
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
1758
|
+
}
|
|
1759
|
+
function parseJsonRecord(value) {
|
|
1760
|
+
if (!value)
|
|
1761
|
+
return {};
|
|
1762
|
+
try {
|
|
1763
|
+
const parsed = JSON.parse(value);
|
|
1764
|
+
return objectRecord3(parsed);
|
|
1765
|
+
} catch {
|
|
1766
|
+
return {};
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
function stringAttribute(attrs, key) {
|
|
1770
|
+
const value = attrs[key];
|
|
1771
|
+
if (typeof value === "string")
|
|
1772
|
+
return value;
|
|
1773
|
+
if (typeof value === "number" || typeof value === "bigint")
|
|
1774
|
+
return String(value);
|
|
1775
|
+
return null;
|
|
1776
|
+
}
|
|
1777
|
+
function numberAttribute(attrs, key) {
|
|
1778
|
+
const value = attrs[key];
|
|
1779
|
+
if (typeof value === "number" && Number.isFinite(value))
|
|
1780
|
+
return value;
|
|
1781
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
1782
|
+
const parsed = Number(value);
|
|
1783
|
+
if (Number.isFinite(parsed))
|
|
1784
|
+
return parsed;
|
|
1785
|
+
}
|
|
1786
|
+
return null;
|
|
1787
|
+
}
|
|
1788
|
+
function commandString(value) {
|
|
1789
|
+
if (typeof value === "string")
|
|
1790
|
+
return value;
|
|
1791
|
+
if (Array.isArray(value)) {
|
|
1792
|
+
const parts = value.map((item) => typeof item === "string" || typeof item === "number" || typeof item === "bigint" ? String(item) : null);
|
|
1793
|
+
return parts.every(Boolean) ? parts.join(" ") : null;
|
|
1794
|
+
}
|
|
1795
|
+
return null;
|
|
1796
|
+
}
|
|
1797
|
+
function normalizeLogLevel(value) {
|
|
1798
|
+
if (value === "debug" || value === "info" || value === "warn" || value === "error" || value === "fatal")
|
|
1799
|
+
return value;
|
|
1800
|
+
return "info";
|
|
1801
|
+
}
|
|
1802
|
+
function rowExists(db, table, id) {
|
|
1803
|
+
const row = db.prepare(`SELECT 1 AS found FROM ${table} WHERE id = ? LIMIT 1`).get(id);
|
|
1804
|
+
return Boolean(row);
|
|
1805
|
+
}
|
|
1806
|
+
function sealSegment(db, segmentId) {
|
|
1807
|
+
const segment = db.prepare("SELECT relative_path FROM event_segments WHERE id = ?").get(segmentId);
|
|
1808
|
+
const segmentHash = segment ? sha256(readFileSync(resolveDataPath(db, segment.relative_path))) : null;
|
|
1809
|
+
db.prepare(`
|
|
1810
|
+
UPDATE event_segments
|
|
1811
|
+
SET sealed_at = COALESCE(sealed_at, strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
1812
|
+
segment_hash = COALESCE(?, segment_hash),
|
|
1813
|
+
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
1814
|
+
WHERE id = ?
|
|
1815
|
+
`).run(segmentHash, segmentId);
|
|
1816
|
+
writeSegmentManifest(db, segmentId);
|
|
1817
|
+
}
|
|
1818
|
+
function appendAndFlush(path, bytes) {
|
|
1819
|
+
const fd = openSync(path, "a");
|
|
1820
|
+
try {
|
|
1821
|
+
let written = 0;
|
|
1822
|
+
while (written < bytes.byteLength) {
|
|
1823
|
+
const n = writeSync(fd, bytes, written, bytes.byteLength - written);
|
|
1824
|
+
if (n <= 0)
|
|
1825
|
+
throw new Error(`Failed to append bytes to ${path}`);
|
|
1826
|
+
written += n;
|
|
1827
|
+
}
|
|
1828
|
+
if (shouldFsync()) {
|
|
1829
|
+
fsyncSync(fd);
|
|
1830
|
+
}
|
|
1831
|
+
} finally {
|
|
1832
|
+
closeSync(fd);
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
function withEventStoreLock(db, fn) {
|
|
1836
|
+
const depth = dbLockDepths.get(db) ?? 0;
|
|
1837
|
+
if (depth > 0) {
|
|
1838
|
+
dbLockDepths.set(db, depth + 1);
|
|
1839
|
+
try {
|
|
1840
|
+
return fn();
|
|
1841
|
+
} finally {
|
|
1842
|
+
const nextDepth = (dbLockDepths.get(db) ?? 1) - 1;
|
|
1843
|
+
if (nextDepth > 0)
|
|
1844
|
+
dbLockDepths.set(db, nextDepth);
|
|
1845
|
+
else
|
|
1846
|
+
dbLockDepths.delete(db);
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
const lockRoot = resolveDataPath(db, ".locks");
|
|
1850
|
+
const lockDir = resolveDataPath(db, ".locks/segments.lock");
|
|
1851
|
+
mkdirSync(lockRoot, { recursive: true });
|
|
1852
|
+
const timeoutMs = readPositiveIntEnv("HASNA_LOGS_LOCK_TIMEOUT_MS", 1e4);
|
|
1853
|
+
const staleMs = readPositiveIntEnv("HASNA_LOGS_LOCK_STALE_MS", 120000);
|
|
1854
|
+
const start = Date.now();
|
|
1855
|
+
while (true) {
|
|
1856
|
+
try {
|
|
1857
|
+
mkdirSync(lockDir);
|
|
1858
|
+
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, created_at: new Date().toISOString() })}
|
|
1859
|
+
`, "utf8");
|
|
1860
|
+
break;
|
|
1861
|
+
} catch (error) {
|
|
1862
|
+
if (!isFileExistsError(error))
|
|
1863
|
+
throw error;
|
|
1864
|
+
try {
|
|
1865
|
+
const ageMs = Date.now() - statSync(lockDir).mtimeMs;
|
|
1866
|
+
if (ageMs > staleMs) {
|
|
1867
|
+
rmSync(lockDir, { recursive: true, force: true });
|
|
1868
|
+
continue;
|
|
1869
|
+
}
|
|
1870
|
+
} catch {
|
|
1871
|
+
continue;
|
|
1872
|
+
}
|
|
1873
|
+
if (Date.now() - start > timeoutMs) {
|
|
1874
|
+
throw new Error(`Timed out waiting for event store lock: ${lockDir}`);
|
|
1875
|
+
}
|
|
1876
|
+
sleepSync(20);
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
try {
|
|
1880
|
+
dbLockDepths.set(db, 1);
|
|
1881
|
+
return fn();
|
|
1882
|
+
} finally {
|
|
1883
|
+
dbLockDepths.delete(db);
|
|
1884
|
+
rmSync(lockDir, { recursive: true, force: true });
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
function readPositiveIntEnv(name, fallback) {
|
|
1888
|
+
const parsed = Number.parseInt(process.env[name] ?? "", 10);
|
|
1889
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
1890
|
+
}
|
|
1891
|
+
function isFileExistsError(error) {
|
|
1892
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "EEXIST");
|
|
1893
|
+
}
|
|
1894
|
+
function sleepSync(ms) {
|
|
1895
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
1896
|
+
}
|
|
1897
|
+
function getMaxSegmentBytes() {
|
|
1898
|
+
const configured = process.env.HASNA_LOGS_SEGMENT_MAX_BYTES ?? process.env.OPEN_LOGS_SEGMENT_MAX_BYTES;
|
|
1899
|
+
const parsed = configured ? Number.parseInt(configured, 10) : 64 * 1024 * 1024;
|
|
1900
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 64 * 1024 * 1024;
|
|
1901
|
+
}
|
|
1902
|
+
function shouldFsync() {
|
|
1903
|
+
return process.env.HASNA_LOGS_FSYNC !== "0" && process.env.OPEN_LOGS_FSYNC !== "0";
|
|
1904
|
+
}
|
|
1905
|
+
function shouldHashSegmentOnAppend() {
|
|
1906
|
+
return process.env.HASNA_LOGS_SEGMENT_HASH_ON_APPEND !== "0" && process.env.OPEN_LOGS_SEGMENT_HASH_ON_APPEND !== "0";
|
|
1907
|
+
}
|
|
1908
|
+
function resolveDataPath(db, relativePath) {
|
|
1909
|
+
const root = resolve(getEventStoreDataDir(db));
|
|
1910
|
+
const fullPath = resolve(root, ...relativePath.split("/"));
|
|
1911
|
+
if (fullPath !== root && !fullPath.startsWith(root + sep)) {
|
|
1912
|
+
throw new Error(`Refusing to access path outside event store: ${relativePath}`);
|
|
1913
|
+
}
|
|
1914
|
+
return fullPath;
|
|
1915
|
+
}
|
|
1916
|
+
function sha256(input) {
|
|
1917
|
+
return createHash4("sha256").update(input).digest("hex");
|
|
1918
|
+
}
|
|
1919
|
+
function errorMessage(error) {
|
|
1920
|
+
return error instanceof Error ? error.message : String(error);
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
// src/db/index.ts
|
|
1924
|
+
import { Database } from "bun:sqlite";
|
|
1925
|
+
import { createHash as createHash5 } from "crypto";
|
|
1926
|
+
import { cpSync, existsSync as existsSync2, mkdirSync as mkdirSync2, mkdtempSync } from "fs";
|
|
1927
|
+
import { join as join2 } from "path";
|
|
1928
|
+
|
|
1929
|
+
// src/db/migrations/001_alert_rules.ts
|
|
1930
|
+
function migrateAlertRules(db) {
|
|
1931
|
+
db.run(`
|
|
1932
|
+
CREATE TABLE IF NOT EXISTS alert_rules (
|
|
1933
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
1934
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
1935
|
+
name TEXT NOT NULL,
|
|
1936
|
+
service TEXT,
|
|
1937
|
+
level TEXT NOT NULL DEFAULT 'error' CHECK(level IN ('debug','info','warn','error','fatal')),
|
|
1938
|
+
threshold_count INTEGER NOT NULL DEFAULT 10,
|
|
1939
|
+
window_seconds INTEGER NOT NULL DEFAULT 60,
|
|
1940
|
+
action TEXT NOT NULL DEFAULT 'webhook' CHECK(action IN ('webhook','log')),
|
|
1941
|
+
webhook_url TEXT,
|
|
1942
|
+
enabled INTEGER NOT NULL DEFAULT 1,
|
|
1943
|
+
last_fired_at TEXT,
|
|
1944
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
1945
|
+
)
|
|
1946
|
+
`);
|
|
1947
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_alert_rules_project ON alert_rules(project_id)");
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
// src/db/migrations/002_issues.ts
|
|
1951
|
+
function migrateIssues(db) {
|
|
1952
|
+
db.run(`
|
|
1953
|
+
CREATE TABLE IF NOT EXISTS issues (
|
|
1954
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
1955
|
+
project_id TEXT REFERENCES projects(id) ON DELETE CASCADE,
|
|
1956
|
+
fingerprint TEXT NOT NULL,
|
|
1957
|
+
level TEXT NOT NULL,
|
|
1958
|
+
service TEXT,
|
|
1959
|
+
message_template TEXT NOT NULL,
|
|
1960
|
+
first_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
1961
|
+
last_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
1962
|
+
count INTEGER NOT NULL DEFAULT 1,
|
|
1963
|
+
status TEXT NOT NULL DEFAULT 'open' CHECK(status IN ('open','resolved','ignored')),
|
|
1964
|
+
UNIQUE(project_id, fingerprint)
|
|
1965
|
+
)
|
|
1966
|
+
`);
|
|
1967
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_issues_project ON issues(project_id, status)");
|
|
1968
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_issues_fingerprint ON issues(fingerprint)");
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
// src/db/migrations/003_retention.ts
|
|
1972
|
+
var RETENTION_COLUMNS = [
|
|
1973
|
+
"max_rows INTEGER NOT NULL DEFAULT 100000",
|
|
1974
|
+
"debug_ttl_hours INTEGER NOT NULL DEFAULT 24",
|
|
1975
|
+
"info_ttl_hours INTEGER NOT NULL DEFAULT 168",
|
|
1976
|
+
"warn_ttl_hours INTEGER NOT NULL DEFAULT 720",
|
|
1977
|
+
"error_ttl_hours INTEGER NOT NULL DEFAULT 2160"
|
|
1978
|
+
];
|
|
1979
|
+
function migrateRetention(db) {
|
|
1980
|
+
for (const col of RETENTION_COLUMNS) {
|
|
1981
|
+
try {
|
|
1982
|
+
db.run(`ALTER TABLE projects ADD COLUMN ${col}`);
|
|
1983
|
+
} catch {}
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
// src/db/migrations/004_page_auth.ts
|
|
1988
|
+
function migratePageAuth(db) {
|
|
1989
|
+
db.run(`
|
|
1990
|
+
CREATE TABLE IF NOT EXISTS page_auth (
|
|
1991
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
1992
|
+
page_id TEXT NOT NULL UNIQUE REFERENCES pages(id) ON DELETE CASCADE,
|
|
1993
|
+
type TEXT NOT NULL CHECK(type IN ('cookie','bearer','basic')),
|
|
1994
|
+
credentials TEXT NOT NULL,
|
|
1995
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
1996
|
+
)
|
|
1997
|
+
`);
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// src/db/index.ts
|
|
2001
|
+
function resolveDataDir() {
|
|
2002
|
+
const explicit = process.env.HASNA_LOGS_DATA_DIR ?? process.env.LOGS_DATA_DIR;
|
|
2003
|
+
if (explicit)
|
|
2004
|
+
return explicit;
|
|
2005
|
+
const home = process.env.HOME ?? "~";
|
|
2006
|
+
const newDir = join2(home, ".hasna", "logs");
|
|
2007
|
+
const oldDir = join2(home, ".logs");
|
|
2008
|
+
if (!existsSync2(newDir) && existsSync2(oldDir)) {
|
|
2009
|
+
mkdirSync2(join2(home, ".hasna"), { recursive: true });
|
|
2010
|
+
cpSync(oldDir, newDir, { recursive: true });
|
|
2011
|
+
}
|
|
2012
|
+
return newDir;
|
|
2013
|
+
}
|
|
2014
|
+
var DATA_DIR = resolveDataDir();
|
|
2015
|
+
var DB_PATH = process.env.HASNA_LOGS_DB_PATH ?? process.env.LOGS_DB_PATH ?? join2(DATA_DIR, "logs.db");
|
|
2016
|
+
var _db = null;
|
|
2017
|
+
function getDb() {
|
|
2018
|
+
if (_db)
|
|
2019
|
+
return _db;
|
|
2020
|
+
if (!existsSync2(DATA_DIR))
|
|
2021
|
+
mkdirSync2(DATA_DIR, { recursive: true });
|
|
2022
|
+
_db = new Database(DB_PATH);
|
|
2023
|
+
setEventStoreDataDir(_db, DATA_DIR);
|
|
2024
|
+
configureDb(_db);
|
|
2025
|
+
runWithBusyRetry(_db, `CREATE TABLE IF NOT EXISTS feedback (
|
|
2026
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
|
2027
|
+
message TEXT NOT NULL,
|
|
2028
|
+
email TEXT,
|
|
2029
|
+
category TEXT DEFAULT 'general',
|
|
2030
|
+
version TEXT,
|
|
2031
|
+
machine_id TEXT,
|
|
2032
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
2033
|
+
)`);
|
|
2034
|
+
return _db;
|
|
2035
|
+
}
|
|
2036
|
+
function configureDb(db) {
|
|
2037
|
+
db.run("PRAGMA busy_timeout=10000");
|
|
2038
|
+
runWithBusyRetry(db, "PRAGMA journal_mode=WAL");
|
|
2039
|
+
runWithBusyRetry(db, "PRAGMA foreign_keys=ON");
|
|
2040
|
+
migrate(db);
|
|
2041
|
+
}
|
|
2042
|
+
function migrate(db) {
|
|
2043
|
+
runWithBusyRetry(db, `
|
|
2044
|
+
CREATE TABLE IF NOT EXISTS projects (
|
|
2045
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
2046
|
+
name TEXT NOT NULL UNIQUE,
|
|
2047
|
+
github_repo TEXT,
|
|
2048
|
+
base_url TEXT,
|
|
2049
|
+
description TEXT,
|
|
2050
|
+
github_description TEXT,
|
|
2051
|
+
github_branch TEXT,
|
|
2052
|
+
github_sha TEXT,
|
|
2053
|
+
last_synced_at TEXT,
|
|
2054
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2055
|
+
)
|
|
2056
|
+
`);
|
|
2057
|
+
runWithBusyRetry(db, `
|
|
2058
|
+
CREATE TABLE IF NOT EXISTS pages (
|
|
2059
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
2060
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
2061
|
+
url TEXT NOT NULL,
|
|
2062
|
+
path TEXT NOT NULL DEFAULT '/',
|
|
2063
|
+
name TEXT,
|
|
2064
|
+
last_scanned_at TEXT,
|
|
2065
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2066
|
+
UNIQUE(project_id, url)
|
|
2067
|
+
)
|
|
2068
|
+
`);
|
|
2069
|
+
runWithBusyRetry(db, `
|
|
2070
|
+
CREATE TABLE IF NOT EXISTS browser_ingest_tokens (
|
|
2071
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
|
2072
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
2073
|
+
token_hash TEXT NOT NULL UNIQUE,
|
|
2074
|
+
token_prefix TEXT NOT NULL,
|
|
2075
|
+
name TEXT,
|
|
2076
|
+
allowed_origins TEXT,
|
|
2077
|
+
enabled INTEGER NOT NULL DEFAULT 1,
|
|
2078
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2079
|
+
last_used_at TEXT
|
|
2080
|
+
)
|
|
2081
|
+
`);
|
|
2082
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_browser_ingest_tokens_project ON browser_ingest_tokens(project_id, enabled)");
|
|
2083
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_browser_ingest_tokens_hash ON browser_ingest_tokens(token_hash)");
|
|
2084
|
+
runWithBusyRetry(db, `
|
|
2085
|
+
CREATE TABLE IF NOT EXISTS logs (
|
|
2086
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
|
2087
|
+
timestamp TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2088
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2089
|
+
page_id TEXT REFERENCES pages(id) ON DELETE SET NULL,
|
|
2090
|
+
level TEXT NOT NULL CHECK(level IN ('debug','info','warn','error','fatal')),
|
|
2091
|
+
source TEXT NOT NULL DEFAULT 'sdk',
|
|
2092
|
+
service TEXT,
|
|
2093
|
+
message TEXT NOT NULL,
|
|
2094
|
+
trace_id TEXT,
|
|
2095
|
+
session_id TEXT,
|
|
2096
|
+
agent TEXT,
|
|
2097
|
+
url TEXT,
|
|
2098
|
+
stack_trace TEXT,
|
|
2099
|
+
metadata TEXT
|
|
2100
|
+
)
|
|
2101
|
+
`);
|
|
2102
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_logs_project_level_ts ON logs(project_id, level, timestamp DESC)");
|
|
2103
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_logs_trace ON logs(trace_id)");
|
|
2104
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_logs_service ON logs(service)");
|
|
2105
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_logs_page ON logs(page_id)");
|
|
2106
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp DESC)");
|
|
2107
|
+
db.run(`
|
|
2108
|
+
CREATE TABLE IF NOT EXISTS machines (
|
|
2109
|
+
id TEXT PRIMARY KEY,
|
|
2110
|
+
hostname TEXT,
|
|
2111
|
+
platform TEXT,
|
|
2112
|
+
arch TEXT,
|
|
2113
|
+
os_release TEXT,
|
|
2114
|
+
metadata TEXT,
|
|
2115
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2116
|
+
last_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2117
|
+
)
|
|
2118
|
+
`);
|
|
2119
|
+
db.run(`
|
|
2120
|
+
CREATE TABLE IF NOT EXISTS repositories (
|
|
2121
|
+
id TEXT PRIMARY KEY,
|
|
2122
|
+
root_path TEXT,
|
|
2123
|
+
remote_url TEXT,
|
|
2124
|
+
branch TEXT,
|
|
2125
|
+
commit_sha TEXT,
|
|
2126
|
+
dirty INTEGER,
|
|
2127
|
+
metadata TEXT,
|
|
2128
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2129
|
+
last_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2130
|
+
)
|
|
2131
|
+
`);
|
|
2132
|
+
db.run(`
|
|
2133
|
+
CREATE TABLE IF NOT EXISTS apps (
|
|
2134
|
+
id TEXT PRIMARY KEY,
|
|
2135
|
+
repo_id TEXT,
|
|
2136
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2137
|
+
name TEXT,
|
|
2138
|
+
runtime TEXT,
|
|
2139
|
+
environment TEXT,
|
|
2140
|
+
version TEXT,
|
|
2141
|
+
metadata TEXT,
|
|
2142
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2143
|
+
last_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2144
|
+
)
|
|
2145
|
+
`);
|
|
2146
|
+
db.run(`
|
|
2147
|
+
CREATE TABLE IF NOT EXISTS processes (
|
|
2148
|
+
id TEXT PRIMARY KEY,
|
|
2149
|
+
machine_id TEXT,
|
|
2150
|
+
repo_id TEXT,
|
|
2151
|
+
app_id TEXT,
|
|
2152
|
+
pid INTEGER,
|
|
2153
|
+
ppid INTEGER,
|
|
2154
|
+
command TEXT,
|
|
2155
|
+
cwd TEXT,
|
|
2156
|
+
started_at TEXT,
|
|
2157
|
+
ended_at TEXT,
|
|
2158
|
+
exit_code INTEGER,
|
|
2159
|
+
metadata TEXT,
|
|
2160
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2161
|
+
)
|
|
2162
|
+
`);
|
|
2163
|
+
db.run(`
|
|
2164
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
2165
|
+
id TEXT PRIMARY KEY,
|
|
2166
|
+
process_id TEXT,
|
|
2167
|
+
run_type TEXT,
|
|
2168
|
+
name TEXT,
|
|
2169
|
+
status TEXT,
|
|
2170
|
+
started_at TEXT,
|
|
2171
|
+
ended_at TEXT,
|
|
2172
|
+
exit_code INTEGER,
|
|
2173
|
+
metadata TEXT,
|
|
2174
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2175
|
+
)
|
|
2176
|
+
`);
|
|
2177
|
+
db.run(`
|
|
2178
|
+
CREATE TABLE IF NOT EXISTS event_sources (
|
|
2179
|
+
id TEXT PRIMARY KEY,
|
|
2180
|
+
source_type TEXT NOT NULL,
|
|
2181
|
+
name TEXT,
|
|
2182
|
+
version TEXT,
|
|
2183
|
+
config_hash TEXT,
|
|
2184
|
+
metadata TEXT,
|
|
2185
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2186
|
+
last_seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2187
|
+
)
|
|
2188
|
+
`);
|
|
2189
|
+
db.run(`
|
|
2190
|
+
CREATE TABLE IF NOT EXISTS traces (
|
|
2191
|
+
id TEXT PRIMARY KEY,
|
|
2192
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2193
|
+
app_id TEXT,
|
|
2194
|
+
root_span_id TEXT,
|
|
2195
|
+
started_at TEXT,
|
|
2196
|
+
ended_at TEXT,
|
|
2197
|
+
status TEXT,
|
|
2198
|
+
metadata TEXT,
|
|
2199
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2200
|
+
)
|
|
2201
|
+
`);
|
|
2202
|
+
db.run(`
|
|
2203
|
+
CREATE TABLE IF NOT EXISTS spans (
|
|
2204
|
+
id TEXT PRIMARY KEY,
|
|
2205
|
+
trace_id TEXT,
|
|
2206
|
+
parent_span_id TEXT,
|
|
2207
|
+
app_id TEXT,
|
|
2208
|
+
process_id TEXT,
|
|
2209
|
+
name TEXT,
|
|
2210
|
+
operation TEXT,
|
|
2211
|
+
status TEXT,
|
|
2212
|
+
started_at TEXT,
|
|
2213
|
+
ended_at TEXT,
|
|
2214
|
+
duration_ms REAL,
|
|
2215
|
+
metadata TEXT,
|
|
2216
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2217
|
+
)
|
|
2218
|
+
`);
|
|
2219
|
+
db.run(`
|
|
2220
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
2221
|
+
id TEXT PRIMARY KEY,
|
|
2222
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2223
|
+
app_id TEXT,
|
|
2224
|
+
user_hash TEXT,
|
|
2225
|
+
started_at TEXT,
|
|
2226
|
+
ended_at TEXT,
|
|
2227
|
+
status TEXT,
|
|
2228
|
+
metadata TEXT,
|
|
2229
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2230
|
+
)
|
|
2231
|
+
`);
|
|
2232
|
+
db.run(`
|
|
2233
|
+
CREATE TABLE IF NOT EXISTS releases (
|
|
2234
|
+
id TEXT PRIMARY KEY,
|
|
2235
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2236
|
+
app_id TEXT,
|
|
2237
|
+
version TEXT,
|
|
2238
|
+
commit_sha TEXT,
|
|
2239
|
+
build_id TEXT,
|
|
2240
|
+
deployed_at TEXT,
|
|
2241
|
+
metadata TEXT,
|
|
2242
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2243
|
+
)
|
|
2244
|
+
`);
|
|
2245
|
+
db.run(`
|
|
2246
|
+
CREATE TABLE IF NOT EXISTS artifacts (
|
|
2247
|
+
id TEXT PRIMARY KEY,
|
|
2248
|
+
release_id TEXT,
|
|
2249
|
+
artifact_type TEXT,
|
|
2250
|
+
path TEXT,
|
|
2251
|
+
content_hash TEXT,
|
|
2252
|
+
size_bytes INTEGER,
|
|
2253
|
+
metadata TEXT,
|
|
2254
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2255
|
+
)
|
|
2256
|
+
`);
|
|
2257
|
+
db.run(`
|
|
2258
|
+
CREATE TABLE IF NOT EXISTS source_maps (
|
|
2259
|
+
id TEXT PRIMARY KEY,
|
|
2260
|
+
event_id TEXT REFERENCES event_records(event_id) ON DELETE CASCADE,
|
|
2261
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2262
|
+
machine_id TEXT,
|
|
2263
|
+
repo_id TEXT,
|
|
2264
|
+
app_id TEXT,
|
|
2265
|
+
process_id TEXT,
|
|
2266
|
+
run_id TEXT,
|
|
2267
|
+
environment TEXT,
|
|
2268
|
+
source_map_artifact_id TEXT,
|
|
2269
|
+
javascript_artifact_id TEXT,
|
|
2270
|
+
source_map_path TEXT,
|
|
2271
|
+
javascript_path TEXT,
|
|
2272
|
+
source_root TEXT,
|
|
2273
|
+
file TEXT,
|
|
2274
|
+
version INTEGER,
|
|
2275
|
+
validation_status TEXT,
|
|
2276
|
+
validation_error TEXT,
|
|
2277
|
+
source_count INTEGER,
|
|
2278
|
+
names_count INTEGER,
|
|
2279
|
+
mappings_length INTEGER,
|
|
2280
|
+
has_sources_content INTEGER,
|
|
2281
|
+
truncated INTEGER,
|
|
2282
|
+
content_hash TEXT,
|
|
2283
|
+
size_bytes INTEGER,
|
|
2284
|
+
metadata TEXT,
|
|
2285
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2286
|
+
)
|
|
2287
|
+
`);
|
|
2288
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_source_maps_run ON source_maps(run_id, created_at DESC)");
|
|
2289
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_source_maps_js_path ON source_maps(javascript_path)");
|
|
2290
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_source_maps_status ON source_maps(validation_status)");
|
|
2291
|
+
db.run(`
|
|
2292
|
+
CREATE TABLE IF NOT EXISTS source_map_sources (
|
|
2293
|
+
id TEXT PRIMARY KEY,
|
|
2294
|
+
source_map_id TEXT NOT NULL REFERENCES source_maps(id) ON DELETE CASCADE,
|
|
2295
|
+
ordinal INTEGER NOT NULL,
|
|
2296
|
+
source_path TEXT,
|
|
2297
|
+
has_content INTEGER,
|
|
2298
|
+
content_hash TEXT,
|
|
2299
|
+
metadata TEXT,
|
|
2300
|
+
UNIQUE(source_map_id, ordinal)
|
|
2301
|
+
)
|
|
2302
|
+
`);
|
|
2303
|
+
migrateSourceMapSourcesSyncId(db);
|
|
2304
|
+
db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_source_map_sources_id ON source_map_sources(id)");
|
|
2305
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_source_map_sources_path ON source_map_sources(source_path)");
|
|
2306
|
+
db.run(`
|
|
2307
|
+
CREATE TABLE IF NOT EXISTS test_reports (
|
|
2308
|
+
id TEXT PRIMARY KEY,
|
|
2309
|
+
event_id TEXT REFERENCES event_records(event_id) ON DELETE CASCADE,
|
|
2310
|
+
source_event_id TEXT,
|
|
2311
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2312
|
+
machine_id TEXT,
|
|
2313
|
+
repo_id TEXT,
|
|
2314
|
+
app_id TEXT,
|
|
2315
|
+
process_id TEXT,
|
|
2316
|
+
run_id TEXT,
|
|
2317
|
+
environment TEXT,
|
|
2318
|
+
source TEXT,
|
|
2319
|
+
event_time TEXT,
|
|
2320
|
+
path TEXT,
|
|
2321
|
+
format TEXT,
|
|
2322
|
+
parser TEXT,
|
|
2323
|
+
parse_status TEXT,
|
|
2324
|
+
parse_error TEXT,
|
|
2325
|
+
size_bytes INTEGER,
|
|
2326
|
+
content_hash TEXT,
|
|
2327
|
+
changed TEXT,
|
|
2328
|
+
mtime_ms REAL,
|
|
2329
|
+
tests INTEGER,
|
|
2330
|
+
failures INTEGER,
|
|
2331
|
+
errors INTEGER,
|
|
2332
|
+
skipped INTEGER,
|
|
2333
|
+
time_seconds REAL,
|
|
2334
|
+
suite_count INTEGER,
|
|
2335
|
+
testcase_count INTEGER,
|
|
2336
|
+
case_stored_count INTEGER NOT NULL DEFAULT 0,
|
|
2337
|
+
truncated INTEGER NOT NULL DEFAULT 0,
|
|
2338
|
+
metadata TEXT,
|
|
2339
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2340
|
+
)
|
|
2341
|
+
`);
|
|
2342
|
+
db.run(`
|
|
2343
|
+
CREATE TABLE IF NOT EXISTS test_cases (
|
|
2344
|
+
id TEXT PRIMARY KEY,
|
|
2345
|
+
report_id TEXT NOT NULL REFERENCES test_reports(id) ON DELETE CASCADE,
|
|
2346
|
+
event_id TEXT REFERENCES event_records(event_id) ON DELETE CASCADE,
|
|
2347
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2348
|
+
machine_id TEXT,
|
|
2349
|
+
repo_id TEXT,
|
|
2350
|
+
app_id TEXT,
|
|
2351
|
+
process_id TEXT,
|
|
2352
|
+
run_id TEXT,
|
|
2353
|
+
environment TEXT,
|
|
2354
|
+
suite_name TEXT,
|
|
2355
|
+
suite_index INTEGER,
|
|
2356
|
+
case_index INTEGER,
|
|
2357
|
+
name TEXT,
|
|
2358
|
+
classname TEXT,
|
|
2359
|
+
file TEXT,
|
|
2360
|
+
status TEXT,
|
|
2361
|
+
time_seconds REAL,
|
|
2362
|
+
metadata TEXT,
|
|
2363
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2364
|
+
)
|
|
2365
|
+
`);
|
|
2366
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_test_reports_run_time ON test_reports(run_id, event_time DESC)");
|
|
2367
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_test_reports_status_time ON test_reports(parse_status, event_time DESC)");
|
|
2368
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_test_reports_path_hash ON test_reports(path, content_hash)");
|
|
2369
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_test_cases_report_status ON test_cases(report_id, status)");
|
|
2370
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_test_cases_run_status ON test_cases(run_id, status)");
|
|
2371
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_test_cases_name ON test_cases(name, classname)");
|
|
2372
|
+
db.run(`
|
|
2373
|
+
CREATE TABLE IF NOT EXISTS projection_offsets (
|
|
2374
|
+
projection_name TEXT PRIMARY KEY,
|
|
2375
|
+
segment_id TEXT,
|
|
2376
|
+
byte_offset INTEGER NOT NULL DEFAULT 0,
|
|
2377
|
+
event_id TEXT,
|
|
2378
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2379
|
+
)
|
|
2380
|
+
`);
|
|
2381
|
+
db.run(`
|
|
2382
|
+
CREATE TABLE IF NOT EXISTS sync_cursors (
|
|
2383
|
+
target_id TEXT PRIMARY KEY,
|
|
2384
|
+
cursor TEXT,
|
|
2385
|
+
last_event_id TEXT,
|
|
2386
|
+
last_segment_id TEXT,
|
|
2387
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2388
|
+
)
|
|
2389
|
+
`);
|
|
2390
|
+
db.run(`
|
|
2391
|
+
CREATE TABLE IF NOT EXISTS event_segments (
|
|
2392
|
+
id TEXT PRIMARY KEY,
|
|
2393
|
+
relative_path TEXT NOT NULL UNIQUE,
|
|
2394
|
+
manifest_path TEXT,
|
|
2395
|
+
byte_length INTEGER NOT NULL DEFAULT 0,
|
|
2396
|
+
event_count INTEGER NOT NULL DEFAULT 0,
|
|
2397
|
+
first_event_time TEXT,
|
|
2398
|
+
last_event_time TEXT,
|
|
2399
|
+
segment_hash TEXT,
|
|
2400
|
+
sealed_at TEXT,
|
|
2401
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2402
|
+
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2403
|
+
)
|
|
2404
|
+
`);
|
|
2405
|
+
ensureColumn(db, "event_segments", "manifest_path", "TEXT");
|
|
2406
|
+
ensureColumn(db, "event_segments", "segment_hash", "TEXT");
|
|
2407
|
+
db.run(`
|
|
2408
|
+
CREATE TABLE IF NOT EXISTS event_records (
|
|
2409
|
+
event_id TEXT PRIMARY KEY,
|
|
2410
|
+
schema_version INTEGER NOT NULL DEFAULT 1,
|
|
2411
|
+
source_event_id TEXT,
|
|
2412
|
+
event_type TEXT NOT NULL,
|
|
2413
|
+
event_time TEXT NOT NULL,
|
|
2414
|
+
ingest_time TEXT NOT NULL,
|
|
2415
|
+
severity TEXT,
|
|
2416
|
+
source TEXT NOT NULL,
|
|
2417
|
+
project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
|
|
2418
|
+
page_id TEXT REFERENCES pages(id) ON DELETE SET NULL,
|
|
2419
|
+
log_id TEXT REFERENCES logs(id) ON DELETE SET NULL,
|
|
2420
|
+
machine_id TEXT,
|
|
2421
|
+
repo_id TEXT,
|
|
2422
|
+
app_id TEXT,
|
|
2423
|
+
process_id TEXT,
|
|
2424
|
+
run_id TEXT,
|
|
2425
|
+
trace_id TEXT,
|
|
2426
|
+
span_id TEXT,
|
|
2427
|
+
parent_span_id TEXT,
|
|
2428
|
+
session_id TEXT,
|
|
2429
|
+
release_id TEXT,
|
|
2430
|
+
environment TEXT,
|
|
2431
|
+
artifact_id TEXT,
|
|
2432
|
+
privacy_tier TEXT,
|
|
2433
|
+
segment_id TEXT NOT NULL REFERENCES event_segments(id) ON DELETE CASCADE,
|
|
2434
|
+
segment_path TEXT NOT NULL,
|
|
2435
|
+
byte_offset INTEGER NOT NULL,
|
|
2436
|
+
byte_length INTEGER NOT NULL,
|
|
2437
|
+
record_hash TEXT NOT NULL,
|
|
2438
|
+
message TEXT,
|
|
2439
|
+
metadata TEXT,
|
|
2440
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2441
|
+
)
|
|
2442
|
+
`);
|
|
2443
|
+
ensureColumn(db, "event_records", "source_event_id", "TEXT");
|
|
2444
|
+
ensureColumn(db, "event_records", "machine_id", "TEXT");
|
|
2445
|
+
ensureColumn(db, "event_records", "repo_id", "TEXT");
|
|
2446
|
+
ensureColumn(db, "event_records", "app_id", "TEXT");
|
|
2447
|
+
ensureColumn(db, "event_records", "process_id", "TEXT");
|
|
2448
|
+
ensureColumn(db, "event_records", "run_id", "TEXT");
|
|
2449
|
+
ensureColumn(db, "event_records", "span_id", "TEXT");
|
|
2450
|
+
ensureColumn(db, "event_records", "parent_span_id", "TEXT");
|
|
2451
|
+
ensureColumn(db, "event_records", "release_id", "TEXT");
|
|
2452
|
+
ensureColumn(db, "event_records", "environment", "TEXT");
|
|
2453
|
+
ensureColumn(db, "event_records", "artifact_id", "TEXT");
|
|
2454
|
+
ensureColumn(db, "event_records", "privacy_tier", "TEXT");
|
|
2455
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_time ON event_records(event_time DESC)");
|
|
2456
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_type_source ON event_records(event_type, source)");
|
|
2457
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_project_time ON event_records(project_id, event_time DESC)");
|
|
2458
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_trace ON event_records(trace_id)");
|
|
2459
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_segment ON event_records(segment_id)");
|
|
2460
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_machine_time ON event_records(machine_id, event_time DESC)");
|
|
2461
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_repo_time ON event_records(repo_id, event_time DESC)");
|
|
2462
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_app_time ON event_records(app_id, event_time DESC)");
|
|
2463
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_process_time ON event_records(process_id, event_time DESC)");
|
|
2464
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_run_time ON event_records(run_id, event_time DESC)");
|
|
2465
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_span ON event_records(span_id)");
|
|
2466
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_release_time ON event_records(release_id, event_time DESC)");
|
|
2467
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_source_event ON event_records(source_event_id)");
|
|
2468
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_event_records_environment ON event_records(environment, event_time DESC)");
|
|
2469
|
+
db.run(`
|
|
2470
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS logs_fts USING fts5(
|
|
2471
|
+
message, service, stack_trace,
|
|
2472
|
+
content=logs, content_rowid=rowid
|
|
2473
|
+
)
|
|
2474
|
+
`);
|
|
2475
|
+
db.run(`
|
|
2476
|
+
CREATE TRIGGER IF NOT EXISTS logs_fts_insert AFTER INSERT ON logs BEGIN
|
|
2477
|
+
INSERT INTO logs_fts(rowid, message, service, stack_trace)
|
|
2478
|
+
VALUES (new.rowid, new.message, new.service, new.stack_trace);
|
|
2479
|
+
END
|
|
2480
|
+
`);
|
|
2481
|
+
db.run(`
|
|
2482
|
+
CREATE TRIGGER IF NOT EXISTS logs_fts_delete AFTER DELETE ON logs BEGIN
|
|
2483
|
+
INSERT INTO logs_fts(logs_fts, rowid, message, service, stack_trace)
|
|
2484
|
+
VALUES ('delete', old.rowid, old.message, old.service, old.stack_trace);
|
|
2485
|
+
END
|
|
2486
|
+
`);
|
|
2487
|
+
db.run(`
|
|
2488
|
+
CREATE TABLE IF NOT EXISTS scan_jobs (
|
|
2489
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
2490
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
2491
|
+
page_id TEXT REFERENCES pages(id) ON DELETE SET NULL,
|
|
2492
|
+
schedule TEXT NOT NULL DEFAULT '*/30 * * * *',
|
|
2493
|
+
enabled INTEGER NOT NULL DEFAULT 1,
|
|
2494
|
+
last_run_at TEXT,
|
|
2495
|
+
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now'))
|
|
2496
|
+
)
|
|
2497
|
+
`);
|
|
2498
|
+
db.run(`
|
|
2499
|
+
CREATE TABLE IF NOT EXISTS scan_runs (
|
|
2500
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
2501
|
+
job_id TEXT NOT NULL REFERENCES scan_jobs(id) ON DELETE CASCADE,
|
|
2502
|
+
page_id TEXT REFERENCES pages(id) ON DELETE SET NULL,
|
|
2503
|
+
started_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2504
|
+
finished_at TEXT,
|
|
2505
|
+
status TEXT NOT NULL DEFAULT 'running' CHECK(status IN ('running','completed','failed')),
|
|
2506
|
+
logs_collected INTEGER NOT NULL DEFAULT 0,
|
|
2507
|
+
errors_found INTEGER NOT NULL DEFAULT 0,
|
|
2508
|
+
perf_score REAL
|
|
2509
|
+
)
|
|
2510
|
+
`);
|
|
2511
|
+
db.run(`
|
|
2512
|
+
CREATE TABLE IF NOT EXISTS performance_snapshots (
|
|
2513
|
+
id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))),
|
|
2514
|
+
timestamp TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
2515
|
+
project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
|
|
2516
|
+
page_id TEXT REFERENCES pages(id) ON DELETE SET NULL,
|
|
2517
|
+
url TEXT NOT NULL,
|
|
2518
|
+
lcp REAL,
|
|
2519
|
+
fcp REAL,
|
|
2520
|
+
cls REAL,
|
|
2521
|
+
tti REAL,
|
|
2522
|
+
ttfb REAL,
|
|
2523
|
+
score REAL,
|
|
2524
|
+
raw_audit TEXT
|
|
2525
|
+
)
|
|
2526
|
+
`);
|
|
2527
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_perf_project_ts ON performance_snapshots(project_id, timestamp DESC)");
|
|
2528
|
+
db.run("CREATE INDEX IF NOT EXISTS idx_perf_page ON performance_snapshots(page_id)");
|
|
2529
|
+
migrateAlertRules(db);
|
|
2530
|
+
migrateIssues(db);
|
|
2531
|
+
migrateRetention(db);
|
|
2532
|
+
migratePageAuth(db);
|
|
2533
|
+
}
|
|
2534
|
+
function migrateSourceMapSourcesSyncId(db) {
|
|
2535
|
+
ensureColumn(db, "source_map_sources", "id", "TEXT");
|
|
2536
|
+
const rows = db.prepare(`
|
|
2537
|
+
SELECT source_map_id, ordinal
|
|
2538
|
+
FROM source_map_sources
|
|
2539
|
+
WHERE id IS NULL OR id LIKE 'srcmap_source_legacy_%'
|
|
2540
|
+
`).all();
|
|
2541
|
+
if (rows.length === 0)
|
|
2542
|
+
return;
|
|
2543
|
+
const update = db.prepare(`
|
|
2544
|
+
UPDATE source_map_sources
|
|
2545
|
+
SET id = ?
|
|
2546
|
+
WHERE source_map_id = ? AND ordinal = ?
|
|
2547
|
+
`);
|
|
2548
|
+
db.transaction((items) => {
|
|
2549
|
+
for (const row of items) {
|
|
2550
|
+
update.run(sourceMapSourceRowId2(row.source_map_id, row.ordinal), row.source_map_id, row.ordinal);
|
|
2551
|
+
}
|
|
2552
|
+
})(rows);
|
|
2553
|
+
}
|
|
2554
|
+
function sourceMapSourceRowId2(sourceMapId, ordinal) {
|
|
2555
|
+
return `srcmap_source_${createHash5("md5").update(sourceMapId).update(":").update(String(ordinal)).digest("hex")}`;
|
|
2556
|
+
}
|
|
2557
|
+
function ensureColumn(db, table, column, definition) {
|
|
2558
|
+
const existing = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
2559
|
+
if (!existing.some((c) => c.name === column)) {
|
|
2560
|
+
runWithBusyRetry(db, `ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
function runWithBusyRetry(db, sql) {
|
|
2564
|
+
const started = Date.now();
|
|
2565
|
+
while (true) {
|
|
2566
|
+
try {
|
|
2567
|
+
db.run(sql);
|
|
2568
|
+
return;
|
|
2569
|
+
} catch (error) {
|
|
2570
|
+
if (!isSqliteBusy(error) || Date.now() - started > 1e4)
|
|
2571
|
+
throw error;
|
|
2572
|
+
sleepSync2(25);
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
}
|
|
2576
|
+
function isSqliteBusy(error) {
|
|
2577
|
+
return Boolean(error && typeof error === "object" && "code" in error && error.code === "SQLITE_BUSY");
|
|
2578
|
+
}
|
|
2579
|
+
function sleepSync2(ms) {
|
|
2580
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
2581
|
+
}
|
|
2582
|
+
|
|
2583
|
+
export { upsertIssue, listIssues, getIssue, updateIssueStatus, sanitizeSourceMapTelemetry, sanitizeSourceMapArtifactRecord, sanitizeSourceMapPathValue, sanitizeSourceMapIdentifierValue, sourceMapFallbackIdentifier, sanitizeSourceMapContextRecord, upsertSourceMapProjection, sanitizedTestReportMetadata, upsertTestReportProjection, getEventStoreDataDir, appendRawEvent, indexRawEvent, getEventRecord, readRawEvent, verifyEventStore, rebuildEventStoreIndex, repairEventStoreSegments, withEventStoreLock, getDb };
|