@redthreadlabs/tracelog-schema 0.1.0

package/README.md

@@ -0,0 +1,33 @@
+# @redthreadlabs/tracelog-schema
+
+The shared **contract** for the tracelog suite — one dependency-free, isomorphic
+source of truth so the agent (writer), the client SDK, the server, and the
+viewer (reader) never drift.
+
+It contains only the contract: types and pure functions, no I/O.
+
+- **Record kinds** (`kinds.ts`) — the top-level kinds of each NDJSON line
+  (`transaction`, `span`, `error`, `event`, `metricset`).
+- **S3 key layout** (`keys.ts`) — `buildKey` / `parseKey` (exact inverses) plus
+  `intervalSpan`, `overlapsRange`, `dedupeCurrents`, `normalizeHost`. The layout
+  `{channel}/{interval}/{host}[_{seq}][_current].jsonl[.gz]` is fixed.
+- **Metadata sidecar** (`sidecar.ts`) — the `SidecarMeta` shape written at
+  `<logkey>.meta.json`, plus the `MetaAccumulator` that derives it from a file's
+  bytes (uncompressed size, record count, and an hourly interval×kind histogram).
+- **Wire format** (`wire.ts`) — the `POST /logs` ingest types (`LogBatch`,
+  `LogEventItem`, `TimerItem`, `ClientInfo`).
+
+## Usage
+
+```ts
+import { parseKey, MetaAccumulator, RECORD_KINDS, type LogEventItem } from '@redthreadlabs/tracelog-schema';
+```
+
+## Build / test
+
+Plain `tsc` to CommonJS `dist/`; tests run against the compiled output.
+
+```
+npm run build
+npm test
+```

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * @redthreadlabs/tracelog-schema — the shared contract for the tracelog suite:
+ * record kinds, the S3 key layout, the metadata sidecar, and the `/logs` wire
+ * format. One dependency-free, isomorphic source of truth, so the agent
+ * (writer), the client SDK, the server, and the viewer (reader) never drift.
+ */
+export * from './kinds';
+export * from './keys';
+export * from './sidecar';
+export * from './wire';

package/dist/index.js

@@ -0,0 +1,30 @@
+"use strict";
+/*
+ * Copyright Red Thread Labs LLC. All rights reserved.
+ * Licensed under the BSD 2-Clause License.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * @redthreadlabs/tracelog-schema — the shared contract for the tracelog suite:
+ * record kinds, the S3 key layout, the metadata sidecar, and the `/logs` wire
+ * format. One dependency-free, isomorphic source of truth, so the agent
+ * (writer), the client SDK, the server, and the viewer (reader) never drift.
+ */
+__exportStar(require("./kinds"), exports);
+__exportStar(require("./keys"), exports);
+__exportStar(require("./sidecar"), exports);
+__exportStar(require("./wire"), exports);

package/dist/keys.d.ts

@@ -0,0 +1,66 @@
+/**
+ * The S3 key layout — the contract between the writer (the tracelog agent)
+ * and any reader (the viewer). It is FIXED, not configurable:
+ *
+ *   {channel}/{interval}/{host}[_{seq}][_current].jsonl[.gz]
+ *
+ *   server/2026-06-11/172.31.27.225.jsonl.gz
+ *   server/2026-06-11/172.31.27.225_current.jsonl.gz
+ *   server/2026-06-11/172.31.27.225_1.jsonl.gz
+ *
+ * Channel comes before interval so prefix-scoped lifecycle rules work and a
+ * date-range scan within a channel is one lexicographically-ordered listing.
+ * The basename is underscore-delimited (hostnames cannot contain underscores):
+ * host, then a numeric size-rotation seq when > 0, then the literal 'current'
+ * for the live snapshot. `buildKey` and `parseKey` are exact inverses so the
+ * agent and viewer never drift on this layout.
+ */
+export interface KeyVars {
+    channel: string;
+    interval: string;
+    host: string;
+    /** size-rotation sequence within the interval; 0 (or omitted) = first file */
+    seq?: number;
+    /** the live, still-being-written snapshot */
+    current?: boolean;
+}
+export interface ParsedKey {
+    key: string;
+    channel: string;
+    interval: string;
+    host: string;
+    seq: number;
+    current: boolean;
+    /** compressed object size in bytes, from the listing (0 if unknown) */
+    size: number;
+    lastModified?: Date;
+    etag?: string;
+}
+/** Build a log object's key. Pass `gzip` to append the `.gz` suffix. */
+export declare function buildKey(vars: KeyVars, gzip?: boolean): string;
+/**
+ * Parse a log object's key back into its parts, or null if it is not a log
+ * file in the known grammar (e.g. a sidecar `.meta.json`, or anything else).
+ */
+export declare function parseKey(key: string, size?: number, lastModified?: Date, etag?: string): ParsedKey | null;
+/**
+ * The UTC time span an interval label covers: daily `YYYY-MM-DD` → 24 h,
+ * hourly `YYYY-MM-DDTHH` → 1 h. Unknown grammar → null (callers should be
+ * conservative and keep the file).
+ */
+export declare function intervalSpan(interval: string): [number, number] | null;
+/** Whether a file's interval overlaps [startMs, endMs]. Unknown layout → kept. */
+export declare function overlapsRange(file: Pick<ParsedKey, 'interval'>, startMs: number, endMs: number): boolean;
+/**
+ * Drop `_current` snapshots shadowed by their finalized file: if the finalized
+ * key exists, ignore the (briefly surviving) `_current`. A `_current` with no
+ * finalized sibling is kept — it is either live (today) or a dead host's only
+ * copy.
+ */
+export declare function dedupeCurrents<T extends Pick<ParsedKey, 'channel' | 'interval' | 'host' | 'seq' | 'current'>>(files: T[]): T[];
+/**
+ * Normalize a hostname into the host label used in keys. EC2 internal
+ * hostnames (`ip-A-B-C-D[.…]`) become the dotted IP — which avoids embedding
+ * hyphens in the basename; any other hostname is used as-is.
+ */
+export declare function normalizeHost(hostname: string): string;

package/dist/keys.js

@@ -0,0 +1,108 @@
+"use strict";
+/*
+ * Copyright Red Thread Labs LLC. All rights reserved.
+ * Licensed under the BSD 2-Clause License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildKey = buildKey;
+exports.parseKey = parseKey;
+exports.intervalSpan = intervalSpan;
+exports.overlapsRange = overlapsRange;
+exports.dedupeCurrents = dedupeCurrents;
+exports.normalizeHost = normalizeHost;
+/** Build a log object's key. Pass `gzip` to append the `.gz` suffix. */
+function buildKey(vars, gzip = false) {
+    let basename = vars.host;
+    if (vars.seq && vars.seq > 0)
+        basename += `_${vars.seq}`;
+    if (vars.current)
+        basename += '_current';
+    return `${vars.channel}/${vars.interval}/${basename}.jsonl${gzip ? '.gz' : ''}`;
+}
+/**
+ * Parse a log object's key back into its parts, or null if it is not a log
+ * file in the known grammar (e.g. a sidecar `.meta.json`, or anything else).
+ */
+function parseKey(key, size = 0, lastModified, etag) {
+    const parts = key.split('/');
+    if (parts.length !== 3)
+        return null;
+    const [channel, interval, file] = parts;
+    if (!channel || !interval || !file)
+        return null;
+    let base = file;
+    if (base.endsWith('.gz'))
+        base = base.slice(0, -3);
+    if (!base.endsWith('.jsonl'))
+        return null;
+    base = base.slice(0, -'.jsonl'.length);
+    const segments = base.split('_');
+    const host = segments[0];
+    if (!host)
+        return null;
+    let seq = 0;
+    let current = false;
+    let rest = segments.slice(1);
+    if (rest[rest.length - 1] === 'current') {
+        current = true;
+        rest = rest.slice(0, -1);
+    }
+    if (rest.length === 1 && /^\d+$/.test(rest[0])) {
+        seq = parseInt(rest[0], 10);
+    }
+    else if (rest.length > 0) {
+        return null; // not the grammar we know — ignore, don't fail
+    }
+    return { key, channel, interval, host, seq, current, size, lastModified, etag };
+}
+/**
+ * The UTC time span an interval label covers: daily `YYYY-MM-DD` → 24 h,
+ * hourly `YYYY-MM-DDTHH` → 1 h. Unknown grammar → null (callers should be
+ * conservative and keep the file).
+ */
+function intervalSpan(interval) {
+    let m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(interval);
+    if (m) {
+        const t0 = Date.UTC(+m[1], +m[2] - 1, +m[3]);
+        return [t0, t0 + 86400000];
+    }
+    m = /^(\d{4})-(\d{2})-(\d{2})T(\d{2})$/.exec(interval);
+    if (m) {
+        const t0 = Date.UTC(+m[1], +m[2] - 1, +m[3], +m[4]);
+        return [t0, t0 + 3600000];
+    }
+    return null;
+}
+/** Whether a file's interval overlaps [startMs, endMs]. Unknown layout → kept. */
+function overlapsRange(file, startMs, endMs) {
+    const span = intervalSpan(file.interval);
+    if (!span)
+        return true;
+    return span[0] <= endMs && span[1] > startMs;
+}
+/**
+ * Drop `_current` snapshots shadowed by their finalized file: if the finalized
+ * key exists, ignore the (briefly surviving) `_current`. A `_current` with no
+ * finalized sibling is kept — it is either live (today) or a dead host's only
+ * copy.
+ */
+function dedupeCurrents(files) {
+    const finalized = new Set();
+    for (const f of files) {
+        if (!f.current)
+            finalized.add(`${f.channel}/${f.interval}/${f.host}/${f.seq}`);
+    }
+    return files.filter((f) => !f.current || !finalized.has(`${f.channel}/${f.interval}/${f.host}/${f.seq}`));
+}
+/**
+ * Normalize a hostname into the host label used in keys. EC2 internal
+ * hostnames (`ip-A-B-C-D[.…]`) become the dotted IP — which avoids embedding
+ * hyphens in the basename; any other hostname is used as-is.
+ */
+function normalizeHost(hostname) {
+    const m = /^ip-(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})(\..*)?$/.exec(hostname);
+    if (m) {
+        return `${m[1]}.${m[2]}.${m[3]}.${m[4]}`;
+    }
+    return hostname;
+}

package/dist/kinds.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The record kinds that appear as the single top-level key of every NDJSON
+ * line tracelog writes: `{ "<kind>": { ...fields... } }`. `metadata` is the
+ * once-per-file header line, not a data record, so it is not a RecordKind.
+ */
+export type RecordKind = 'transaction' | 'span' | 'error' | 'event' | 'metricset';
+export declare const RECORD_KINDS: readonly RecordKind[];
+/** The header line's kind. Every file's first line is `{ "metadata": {...} }`. */
+export declare const METADATA_KIND = "metadata";

package/dist/kinds.js

@@ -0,0 +1,16 @@
+"use strict";
+/*
+ * Copyright Red Thread Labs LLC. All rights reserved.
+ * Licensed under the BSD 2-Clause License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.METADATA_KIND = exports.RECORD_KINDS = void 0;
+exports.RECORD_KINDS = [
+    'transaction',
+    'span',
+    'error',
+    'event',
+    'metricset',
+];
+/** The header line's kind. Every file's first line is `{ "metadata": {...} }`. */
+exports.METADATA_KIND = 'metadata';

package/dist/sidecar.d.ts

@@ -0,0 +1,62 @@
+/**
+ * The metadata sidecar: a tiny JSON object written alongside each log object
+ * at `<logkey>.meta.json`, carrying the facts the gzipped body hides so a
+ * reader needs no estimation. `bytes` is the uncompressed size; `intervals`
+ * is an hourly UTC histogram of records by kind. The histogram matters
+ * because buffered remote clients can land past-dated records in today's
+ * file, so a file's nominal interval is a filing label, not a description of
+ * its contents.
+ *
+ *   records === malformed + Σ(all interval/kind counts)
+ */
+export declare const SIDECAR_VERSION = 1;
+export declare const SIDECAR_SUFFIX = ".meta.json";
+export interface SidecarMeta {
+    v: number;
+    /** the file's default (nominal) interval, from its key */
+    interval: string;
+    /** uncompressed byte size */
+    bytes: number;
+    /** compressed (gzipped) object size */
+    compressed: number;
+    /** total records (metadata header line excluded) */
+    records: number;
+    /** records whose timestamp was missing/garbage (not placed in an interval) */
+    malformed: number;
+    /** hourly UTC histogram: { 'YYYY-MM-DDTHH': { kind: count } } */
+    intervals: Record<string, Record<string, number>>;
+}
+/** The sidecar key for a log object key. */
+export declare function sidecarKey(logKey: string): string;
+/** epoch-ms → UTC hour-bucket label 'YYYY-MM-DDTHH'. */
+export declare function hourBucket(ms: number): string;
+/**
+ * Derives a log file's sidecar histogram by parsing its NDJSON lines. The file
+ * on disk is the source of truth: counts come from the exact bytes uploaded,
+ * so they cannot drift from the object, and a restart (which wipes any
+ * write-time counters) or an orphaned file is handled for free by re-deriving.
+ *
+ * Tolerant by design: an unparseable line is skipped (not a record); a record
+ * with a missing/garbage timestamp is counted as `malformed` rather than
+ * forced into an interval. Append-only safe: addChunk may be fed successive
+ * tails of a growing current file, since every line is newline-terminated so
+ * chunk/offset boundaries land between lines.
+ */
+export declare class MetaAccumulator {
+    /** bytes consumed so far (for incremental current-file parsing) */
+    offset: number;
+    records: number;
+    malformed: number;
+    intervals: Record<string, Record<string, number>>;
+    private partial;
+    addChunk(text: string): void;
+    flushPartial(): void;
+    private addLine;
+    /**
+     * The sidecar object for this file. Keys are emitted in a fixed, sorted
+     * order at every level so identical contents serialize to byte-identical
+     * JSON regardless of record arrival order — making a sidecar's ETag a
+     * reliable sameness check.
+     */
+    toMeta(interval: string, bytes: number, compressed: number): SidecarMeta;
+}

package/dist/sidecar.js

@@ -0,0 +1,134 @@
+"use strict";
+/*
+ * Copyright Red Thread Labs LLC. All rights reserved.
+ * Licensed under the BSD 2-Clause License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetaAccumulator = exports.SIDECAR_SUFFIX = exports.SIDECAR_VERSION = void 0;
+exports.sidecarKey = sidecarKey;
+exports.hourBucket = hourBucket;
+const kinds_1 = require("./kinds");
+/**
+ * The metadata sidecar: a tiny JSON object written alongside each log object
+ * at `<logkey>.meta.json`, carrying the facts the gzipped body hides so a
+ * reader needs no estimation. `bytes` is the uncompressed size; `intervals`
+ * is an hourly UTC histogram of records by kind. The histogram matters
+ * because buffered remote clients can land past-dated records in today's
+ * file, so a file's nominal interval is a filing label, not a description of
+ * its contents.
+ *
+ *   records === malformed + Σ(all interval/kind counts)
+ */
+exports.SIDECAR_VERSION = 1;
+exports.SIDECAR_SUFFIX = '.meta.json';
+/** The sidecar key for a log object key. */
+function sidecarKey(logKey) {
+    return logKey + exports.SIDECAR_SUFFIX;
+}
+function pad2(n) {
+    return String(n).padStart(2, '0');
+}
+/** epoch-ms → UTC hour-bucket label 'YYYY-MM-DDTHH'. */
+function hourBucket(ms) {
+    const d = new Date(ms);
+    return (`${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())}` +
+        `T${pad2(d.getUTCHours())}`);
+}
+/**
+ * Derives a log file's sidecar histogram by parsing its NDJSON lines. The file
+ * on disk is the source of truth: counts come from the exact bytes uploaded,
+ * so they cannot drift from the object, and a restart (which wipes any
+ * write-time counters) or an orphaned file is handled for free by re-deriving.
+ *
+ * Tolerant by design: an unparseable line is skipped (not a record); a record
+ * with a missing/garbage timestamp is counted as `malformed` rather than
+ * forced into an interval. Append-only safe: addChunk may be fed successive
+ * tails of a growing current file, since every line is newline-terminated so
+ * chunk/offset boundaries land between lines.
+ */
+class MetaAccumulator {
+    constructor() {
+        /** bytes consumed so far (for incremental current-file parsing) */
+        this.offset = 0;
+        this.records = 0;
+        this.malformed = 0;
+        this.intervals = Object.create(null);
+        this.partial = '';
+    }
+    addChunk(text) {
+        if (!text)
+            return;
+        const s = this.partial + text;
+        let start = 0;
+        let nl;
+        while ((nl = s.indexOf('\n', start)) !== -1) {
+            this.addLine(s.slice(start, nl));
+            start = nl + 1;
+        }
+        this.partial = s.slice(start);
+    }
+    flushPartial() {
+        if (this.partial) {
+            this.addLine(this.partial);
+            this.partial = '';
+        }
+    }
+    addLine(line) {
+        const t = line.trim();
+        if (!t)
+            return;
+        let obj;
+        try {
+            obj = JSON.parse(t);
+        }
+        catch {
+            return; // corrupt line — not a countable record
+        }
+        if (!obj || typeof obj !== 'object')
+            return;
+        const kind = Object.keys(obj)[0];
+        if (!kind || kind === kinds_1.METADATA_KIND)
+            return;
+        this.records++;
+        const body = obj[kind];
+        const tsUs = body &&
+            typeof body.timestamp === 'number' &&
+            isFinite(body.timestamp) &&
+            body.timestamp > 0
+            ? body.timestamp
+            : 0;
+        if (!tsUs) {
+            this.malformed++;
+            return;
+        }
+        const bucket = hourBucket(tsUs / 1000); // serialized timestamps are epoch-µs
+        const byKind = this.intervals[bucket] || (this.intervals[bucket] = Object.create(null));
+        byKind[kind] = (byKind[kind] || 0) + 1;
+    }
+    /**
+     * The sidecar object for this file. Keys are emitted in a fixed, sorted
+     * order at every level so identical contents serialize to byte-identical
+     * JSON regardless of record arrival order — making a sidecar's ETag a
+     * reliable sameness check.
+     */
+    toMeta(interval, bytes, compressed) {
+        const intervals = Object.create(null);
+        for (const hour of Object.keys(this.intervals).sort()) {
+            const src = this.intervals[hour];
+            const sorted = Object.create(null);
+            for (const kind of Object.keys(src).sort())
+                sorted[kind] = src[kind];
+            intervals[hour] = sorted;
+        }
+        return {
+            v: exports.SIDECAR_VERSION,
+            interval,
+            bytes,
+            compressed,
+            records: this.records,
+            malformed: this.malformed,
+            intervals,
+        };
+    }
+}
+exports.MetaAccumulator = MetaAccumulator;

package/dist/wire.d.ts

@@ -0,0 +1,96 @@
+/**
+ * The wire format for the `POST /logs` ingest endpoint: what a remote client
+ * (browser, React Native) sends and what the server parses. The server maps
+ * these into the on-disk record kinds (see kinds.ts). Defined here so the
+ * client SDK, the server, and the viewer all share one definition.
+ */
+export type JsonValue = string | number | boolean | null | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+export interface LogBatch {
+    client: ClientInfo;
+    user_id?: string;
+    session_ref?: string;
+    device_id?: string;
+    events: LogEventItem[];
+    timers: TimerItem[];
+}
+export interface LogEventItem {
+    /** Event category, e.g. 'auth', 'billing', 'startup'. Default: 'client-log' */
+    type: string;
+    /** Epoch milliseconds */
+    timestamp: number;
+    level: LogLevel;
+    message: string;
+    /** Duration in milliseconds (for timed events that aren't span-shaped) */
+    duration?: number;
+    /** Serialized error info. `code` is the structured error code. */
+    error?: {
+        message: string;
+        type?: string;
+        code?: string;
+        stack?: string;
+    };
+    /** Arbitrary key-value event data */
+    params?: Record<string, JsonValue>;
+    /**
+     * Minutes east of UTC at the moment the event was recorded (ISO-8601 sign:
+     * `localWallClock = UTC + tz_offset`). Captured per-event because buffered
+     * clients can record across DST or travel and flush later. e.g. EST = -300,
+     * IST = +330.
+     */
+    tz_offset?: number;
+}
+export interface TimerItem {
+    /** 16-char hex ID, generated client-side */
+    id: string;
+    /** 32-char hex trace ID, shared by parent + children */
+    trace_id: string;
+    /** ID of the root timer in this trace */
+    root_id: string;
+    /** 16-char hex ID of parent timer (absent for root timers) */
+    parent_id?: string;
+    /** Operation name, e.g. 'content-store-startup' */
+    name: string;
+    /** Timer category. Default: 'client-perf' */
+    type: string;
+    /** Start time, epoch milliseconds */
+    timestamp: number;
+    /** Duration in milliseconds */
+    duration: number;
+    outcome: 'success' | 'failure' | 'unknown';
+    context?: {
+        tags?: Record<string, JsonValue>;
+    };
+    /** Minutes east of UTC at record time (see LogEventItem.tz_offset). */
+    tz_offset?: number;
+}
+export interface ClientInfo {
+    /** Application name, e.g. 'duiduidui-app' */
+    name: string;
+    /** Application version */
+    version: string;
+    os: {
+        name: string;
+        version: string;
+    };
+    device: {
+        model?: string;
+        brand?: string;
+        type: string;
+    };
+    runtime: {
+        name: string;
+        version: string;
+    };
+    screen?: {
+        width: number;
+        height: number;
+        pixel_ratio: number;
+    };
+    locale?: string;
+    /** IANA timezone name for the session, e.g. 'America/New_York'. */
+    timezone?: string;
+    device_year_class?: number;
+}

package/dist/wire.js

@@ -0,0 +1,6 @@
+"use strict";
+/*
+ * Copyright Red Thread Labs LLC. All rights reserved.
+ * Licensed under the BSD 2-Clause License.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@redthreadlabs/tracelog-schema",
+  "version": "0.1.0",
+  "description": "Shared contract for the tracelog suite: record kinds, S3 key layout, metadata sidecar, and the /logs wire format",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "BSD-2-Clause",
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build",
+    "pretest": "npm run build",
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "typescript": "latest"
+  }
+}