@kymoscope/sdk 0.0.0-stage → 0.1.1

package/dist/types/lib/importers/mdf/model.d.ts

@@ -0,0 +1,75 @@
+import type { Conv } from './conversion';
+/** A piece of a logical data stream (DT/SD/RD data, or a DZ block to inflate). */
+export interface Fragment {
+    offset: number;
+    length: number;
+    zip: {
+        type: number;
+        param: number;
+        orgLength: number;
+    } | null;
+}
+export interface ChannelModel {
+    name: string;
+    unit: string;
+    /** value: in the record; vlsd: an offset into signal data; master / vmaster: time. */
+    kind: 'value' | 'vlsd' | 'master' | 'vmaster';
+    type: 'uint' | 'int' | 'float' | 'string' | 'bytes';
+    encoding: 'latin1' | 'utf-8' | 'utf-16le' | 'utf-16be';
+    bigEndian: boolean;
+    byteOffset: number;
+    bitOffset: number;
+    bitCount: number;
+    invalBit: number | null;
+    allInvalid: boolean;
+    conv: Conv;
+    /** VLSD: its signal data (SD blocks)… */
+    sd: Fragment[] | null;
+    /** …or the record id of the VLSD channel group holding it. */
+    sdRecordId: number | null;
+}
+export interface GroupModel {
+    name: string;
+    dg: number;
+    recordId: number;
+    recordIdSize: number;
+    /** MDF3 with two record ids: one more id byte after each record. */
+    idTrailer: number;
+    dataBytes: number;
+    invalBytes: number;
+    cycles: number;
+    /** A VLSD channel group: holds another group's strings, not a group of its own. */
+    vlsdCg: boolean;
+    /** The only channel group of its data group: records back to back. */
+    sorted: boolean;
+    master: ChannelModel | null;
+    channels: ChannelModel[];
+    /** The data of its whole data group. */
+    stream: Fragment[];
+    /** Acquisition source bus type (2 = CAN) and the bus-event flag (bus logging, Task 4). */
+    busType: number;
+    busEvent: boolean;
+}
+export interface FileModel {
+    version: string;
+    startTime: string | null;
+    groups: GroupModel[];
+    notes: string[];
+    meta: Record<string, string>;
+}
+/** Bytes of one record: id, data bytes, invalidation bytes, MDF3 trailing id. */
+export declare const recordSize: (g: GroupModel) => number;
+/**
+ * The model as the decoders may trust it (a damaged or hostile file): record
+ * sizes bounded, record counts no more than the data holds, every channel's bits
+ * inside its record and its invalidation bit inside the invalidation bytes.
+ * Channels that are not are skipped with a note (a master that is not: samples
+ * are numbered).
+ */
+export declare function checkModel(model: FileModel, fileSize: number): void;
+/**
+ * Bytes at the start of each record that `channels` and the group's time are read
+ * from (a transposed DZ block inflates only those); all of it when invalidation
+ * bits are read.
+ */
+export declare function bytesNeeded(g: GroupModel, channels: ChannelModel[]): number;

package/dist/types/lib/importers/mdf/records.d.ts

@@ -0,0 +1,31 @@
+import type { BlobReader } from '../blobReader';
+import { type ChannelModel, type FileModel, type GroupModel } from './model';
+/** Records per batch: ~8 MB of record bytes, up to 64k rows (fewer than 1k only for records over 8 KB). */
+export declare const rowsFor: (size: number) => number;
+/** VLSD strings: SD streams read forward, or entries of VLSD channel groups collected while demultiplexing. */
+export declare class Vlsd {
+    #private;
+    constructor(io: BlobReader, signal?: AbortSignal);
+    want(recordId: number): void;
+    wanted(recordId: number): boolean;
+    put(recordId: number, offset: number, bytes: Uint8Array): void;
+    bytes(ch: ChannelModel, offsets: Float64Array): Promise<(Uint8Array | null)[]>;
+    strings(ch: ChannelModel, offsets: Float64Array): Promise<(string | null)[]>;
+}
+/** Time of `n` records from `start` (master, virtual master), or null (no master: sample index). */
+export declare function decodeTime(g: GroupModel, buf: Uint8Array, n: number, start: number, stride?: number, t?: Float64Array): Float64Array | null;
+/**
+ * Raw records of the model groups `wanted` (all in one data group), in file
+ * order, `rows` at a time per group. Sorted: the records back to back.
+ * Unsorted: demultiplexed by record id; VLSD channel group entries go to `vlsd`.
+ * Each yielded buffer is reused once the consumer asks for more. `need`
+ * (sorted): the bytes of each record read (see `bytesNeeded`); `size` is the
+ * record stride of `buf` (shorter than the record when only those are inflated).
+ */
+export declare function rawRecords(io: BlobReader, model: FileModel, wanted: number[], vlsd: Vlsd, signal?: AbortSignal, need?: number): AsyncGenerator<{
+    g: number;
+    buf: Uint8Array;
+    n: number;
+    start: number;
+    size: number;
+}>;

package/dist/types/lib/importers/mdf/stream.d.ts

@@ -0,0 +1,42 @@
+import { type BlobReader } from '../blobReader';
+import type { Fragment } from './model';
+/**
+ * Undo the DZ transposition: `cols` = record size; the bytes past whole rows were
+ * not transposed. `use` < `cols`: only the first `use` bytes of each record, as
+ * records of `use` bytes (`b` then needs only those columns, `use` x rows bytes).
+ */
+export declare function untranspose(b: Uint8Array, cols: number, orgLength: number, use?: number): Uint8Array;
+/**
+ * Bytes per record read from `frags` when only the first `need` bytes of each
+ * `size`-byte record are used: `need` (rounded up to 4 bytes) when every block
+ * is a transposed DZ of whole records (only those columns are inflated), else `size`.
+ */
+export declare function recordStride(frags: Fragment[], size: number, need: number): number;
+/**
+ * The logical bytes of a DZ block, inflated once. `use` < record size (a
+ * transposed block of whole records, see `recordStride`): only the first `use`
+ * bytes of each record, as `use`-byte records. `limit`: a plain block stops once
+ * that many bytes are out (a prefix).
+ */
+export declare function inflateFragment(io: BlobReader, f: Fragment, use?: number, limit?: number): Promise<Uint8Array>;
+/** The logical bytes of a fragment list, window by window (DZ inflated; a large plain-deflate DZ streamed). `use`: see `recordStride`. */
+export declare function logical(io: BlobReader, frags: Fragment[], signal?: AbortSignal, use?: number): AsyncGenerator<Uint8Array>;
+/**
+ * Whole records out of a byte stream, `rows` at a time, at most `limit` records:
+ * views of the stream's own bytes where whole records lie in one piece, else
+ * copied into a buffer that is reused (decode a batch before asking for the next).
+ */
+export declare function records(bytes: AsyncIterable<Uint8Array>, size: number, rows: number, limit: number): AsyncGenerator<{
+    buf: Uint8Array;
+    n: number;
+}>;
+/** Record `index` of an uncompressed fragment list, or null (past the end, or in a DZ block). */
+export declare function recordAt(io: BlobReader, frags: Fragment[], index: number, size: number): Promise<Uint8Array | null>;
+/** VLSD signal data ([u32 length][bytes] entries), read forward through a window. */
+export declare class SdReader {
+    #private;
+    constructor(io: BlobReader, frags: Fragment[], signal?: AbortSignal);
+    /** The entry at `offset` when it is in the window already (no await on the common path). */
+    peek(offset: number): Uint8Array | null;
+    get(offset: number): Promise<Uint8Array>;
+}

package/dist/types/lib/importers/memorySink.d.ts

@@ -0,0 +1,11 @@
+import type { ColumnTable } from '../columnTable';
+import type { Chunk } from './combine';
+/** Combined chunks into one Float64Array per column plus sparse text cells: what the CSV worker returns. */
+export declare class MemorySink {
+    #private;
+    constructor(columns: {
+        name: string;
+    }[], rowsHint: number);
+    append(chunk: Chunk): void;
+    table(): ColumnTable;
+}

package/dist/types/lib/importers/names.d.ts

@@ -0,0 +1,2 @@
+/** Column names unique ignoring case (DuckDB folds case); `reserved` (the time column) is taken. */
+export declare function uniqueNames(names: string[], reserved?: string[]): string[];

package/dist/types/lib/importers/openImport.d.ts

@@ -0,0 +1,33 @@
+import type { LoadedDataset } from '../../types/loadedDataset';
+import { type WorkerLike } from './session';
+import type { ChannelChooser, GroupSelection, Importer } from './types';
+/** Above this many channels the host's chooser (if any) is asked first. */
+export declare const PICKER_CHANNELS = 2000;
+export interface OpenImportOptions {
+    /** Values held by in-memory datasets already (shared budget). */
+    priorInMemoryValues: number;
+    signal?: AbortSignal;
+    onProgress?: (fraction: number) => void;
+    chooseChannels?: ChannelChooser;
+    /** Ask `chooseChannels` whatever the file. */
+    pickChannels?: boolean;
+    /** What the chooser starts from (choosing again for an earlier import). */
+    initial?: GroupSelection[];
+    /** Exactly these channels (split / combine of an earlier import): no chooser. */
+    selection?: GroupSelection[];
+    /** One dataset per group. */
+    split?: boolean;
+    createWorker?: () => WorkerLike;
+    find?: (id: string) => Promise<Importer>;
+}
+/**
+ * A binary file (with its companions) as one dataset, combined on one time
+ * base, or one per group (`split`, or groups without a common time base).
+ * Each output goes in memory while it fits the shared budget, else into DuckDB.
+ */
+export declare function openImport(item: {
+    main: File;
+    companions: File[];
+    format: string;
+    keys?: import('../recordings/keys').RecordingKey[];
+}, o: OpenImportOptions): Promise<LoadedDataset[]>;

package/dist/types/lib/importers/registry.d.ts

@@ -0,0 +1,24 @@
+import { type FormatInfo } from './formats';
+export interface ImportItem {
+    main: File;
+    companions: File[];
+    /** null: delimited text (the CSV path). */
+    format: FormatInfo | null;
+}
+export declare const extOf: (name: string) => string;
+/** The binary format of a file from its first bytes (formats claiming its extension are tried first), or null. */
+export declare function detectFormat(file: File): Promise<FormatInfo | null>;
+/**
+ * Files dropped or picked together, as imports: each binary main file with its
+ * companions (an MF4's .dbc, a COMTRADE .cfg's .dat), delimited text on its
+ * own, and an error for anything else. Order follows `files`.
+ */
+export declare function groupImports(files: File[]): Promise<{
+    items: ImportItem[];
+    errors: {
+        file: string;
+        error: Error;
+    }[];
+}>;
+/** Extensions file pickers and folder trees offer: text, every binary format and their companions. */
+export declare const IMPORTABLE_EXTENSIONS: string[];

package/dist/types/lib/importers/run.d.ts

@@ -0,0 +1,35 @@
+import type { BlobReader } from './blobReader';
+import { type Chunk } from './combine';
+import type { ColumnTable } from '../columnTable';
+import type { FileEvent, GroupSelection, ImportFiles, ImportPlan, Importer } from './types';
+export interface OutColumn {
+    name: string;
+    unit: string;
+    text: boolean;
+}
+export interface ReadResult {
+    columns: OutColumn[];
+    rows: number;
+    /** Memory mode only. */
+    table?: ColumnTable;
+    events: FileEvent[];
+}
+export interface ReadCallbacks {
+    /** Stream mode: one combined chunk; resolve when it is stored (back-pressure). */
+    chunk?: (c: Chunk) => Promise<void>;
+    progress?: (fraction: number) => void;
+    signal?: AbortSignal;
+}
+/** A file's plan, checked (`checkPlan`): the worker and the inline session plan through this. */
+export declare function planImport(importer: Importer, files: ImportFiles, io: BlobReader): Promise<ImportPlan>;
+/** Time, then every selected channel; a name repeated across groups gets its group in brackets. */
+export declare function outColumns(plan: ImportPlan, selection: GroupSelection[]): OutColumn[];
+/** Rows of the combined output (the base group plus every irregular group); -1 when a group's count is unknown. */
+export declare function estimateRows(plan: ImportPlan, selection: GroupSelection[]): number;
+/** Values (rows × columns, Time included) the selection loads; unknown counts assume 4 bytes per value. */
+export declare function estimateValues(plan: ImportPlan, selection: GroupSelection[], fileSize: number): number;
+/**
+ * Read the selected groups of a planned file into one table (memory) or a
+ * stream of chunks (DuckDB), combined on one time base.
+ */
+export declare function readImport(importer: Importer, files: ImportFiles, plan: ImportPlan, io: BlobReader, selection: GroupSelection[], mode: 'memory' | 'stream', on?: ReadCallbacks): Promise<ReadResult>;

package/dist/types/lib/importers/session.d.ts

@@ -0,0 +1,28 @@
+import { type ReadResult } from './run';
+import type { Chunk } from './combine';
+import type { FromWorker, ToWorker } from './workerCore';
+import type { GroupSelection, ImportFiles, ImportPlan, Importer } from './types';
+export interface ReadHandlers {
+    chunk?: (c: Chunk) => Promise<void>;
+    progress?: (fraction: number) => void;
+}
+/** One file being imported: plan once, read one or more selections, then close. */
+export interface ImportSession {
+    plan(): Promise<ImportPlan>;
+    read(selection: GroupSelection[], mode: 'memory' | 'stream', on?: ReadHandlers): Promise<ReadResult>;
+    close(): void;
+}
+/** The slice of the Worker API used here (a fake in tests). */
+export interface WorkerLike {
+    postMessage(m: ToWorker): void;
+    terminate(): void;
+    onmessage: ((e: MessageEvent<FromWorker>) => void) | null;
+    onerror: ((e: unknown) => void) | null;
+}
+/**
+ * Start the next import's worker now; with `h5`, have it load and initialise h5wasm too
+ * (HDF5, MAT 7.3). No-op where imports run inline (no workers).
+ */
+export declare function warmImport(h5: boolean, createWorker?: () => WorkerLike): void;
+/** In a worker where there are workers (always in browsers), else in this thread (tests, server-side). */
+export declare function openSession(importer: string, files: ImportFiles, signal?: AbortSignal, createWorker?: () => WorkerLike, find?: (id: string) => Promise<Importer>): ImportSession;

package/dist/types/lib/importers/tdms/format.d.ts

@@ -0,0 +1,2 @@
+import type { FormatInfo } from '../formats';
+export declare const TDMS_FORMAT: FormatInfo;

package/dist/types/lib/importers/tdms/importer.d.ts

@@ -0,0 +1,10 @@
+import { type Importer } from '../types';
+type Scale = (x: number) => number;
+/**
+ * The NI scale chain of a channel (from the channel, else its group, else the
+ * file, like npTDMS), or null. Linear and polynomial, input sources resolved; a
+ * scale without a type is the DAQmx scaler itself (its raw value).
+ */
+export declare function scaling(chain: Map<string, unknown>[], notes: string[], name: string): Scale | null;
+export declare const tdmsImporter: Importer;
+export {};

package/dist/types/lib/importers/types.d.ts

@@ -0,0 +1,172 @@
+/**
+ * Binary and scientific file importers (wave 1: MF4/MDF3 + DBC, EDF/BDF, XDF,
+ * TDMS, HDF5, MAT, COMTRADE). An importer plans a file from its headers, then
+ * yields batches of samples per group; `readImport` combines the groups onto
+ * one time base and fills a sink (memory or DuckDB).
+ */
+import type { BlobReader } from './blobReader';
+/** How a group's sample times are known. */
+export type TimeBase = 
+/** t = start + i / rate (seconds). */
+{
+    kind: 'rate';
+    rate: number;
+    start: number;
+}
+/** Every batch carries its timestamps (seconds from the start of the file). */
+ | {
+    kind: 'timestamps';
+}
+/** No time in the file: the sample index (such groups cannot be combined). */
+ | {
+    kind: 'index';
+};
+export interface PlanChannel {
+    name: string;
+    unit: string;
+    /** Text values; else numbers. */
+    text?: boolean;
+    /** Combined, it only has values at its own timestamps (markers); else it holds its last value. */
+    sparse?: boolean;
+}
+export interface PlanGroup {
+    name: string;
+    channels: PlanChannel[];
+    /** Samples in the group; -1 when the file does not say before it is read. */
+    samples: number;
+    time: TimeBase;
+    /** Nominal rate in Hz for the UI; null when irregular or unknown. */
+    rateHz: number | null;
+    /** Its timestamps join the combined time base as they are (irregular streams, markers). */
+    irregular?: boolean;
+}
+export interface FileEvent {
+    time: number;
+    text: string;
+}
+export interface ImportPlan {
+    /** Format and version, e.g. 'MF4 4.10', 'EDF+'. */
+    format: string;
+    /** ISO 8601 start of the recording, when the file has one. */
+    startTime: string | null;
+    groups: PlanGroup[];
+    meta: Record<string, string>;
+    events: FileEvent[];
+    /** What was skipped or approximated, in plain English. */
+    notes: string[];
+    /** The structure is ambiguous: ask the host which groups to read (HDF5). */
+    pick?: boolean;
+    /** Groups read when nobody chooses (default: all). */
+    defaultGroups?: number[];
+    /**
+     * The groups share one data stream (unsorted MF4, bus logging): read them in
+     * one pass with `stream`, even when the importer has `streamGroup`.
+     */
+    interleaved?: boolean;
+    /** Importer-private; never leaves the worker. */
+    state: unknown;
+}
+/** Channels to read from one group, in output order (indices into `PlanGroup.channels`). */
+export interface GroupSelection {
+    group: number;
+    channels: number[];
+}
+export type ColumnValues = Float64Array | (string | null)[];
+/**
+ * Samples of one group. Every Float64Array owns its whole buffer (it may be
+ * transferred to the page).
+ */
+export interface Batch {
+    group: number;
+    /** Seconds from the start of the file; null for 'rate' and 'index' groups (made from `start`). */
+    time: Float64Array | null;
+    /** Index of the first row within its group. */
+    start: number;
+    length: number;
+    /** One per selected channel of this group, in selection order. */
+    columns: ColumnValues[];
+    /** Events found while reading (EDF+ annotations). */
+    events?: FileEvent[];
+}
+export interface ImportFiles {
+    main: File;
+    companions: File[];
+    /** A cloud recording's keys (the `krc` importer only): structured-cloned to the worker, never extracted. */
+    keys?: import('../recordings/keys').RecordingKey[];
+}
+export interface StreamArgs {
+    files: ImportFiles;
+    plan: ImportPlan;
+    io: BlobReader;
+    selection: GroupSelection[];
+    signal?: AbortSignal;
+}
+export interface Importer {
+    id: string;
+    label: string;
+    /** Lowercase, with the dot. A hint only: `sniff` decides. */
+    extensions: string[];
+    companions?: {
+        ext: string;
+        anyName?: boolean;
+    }[];
+    sniff(head: Uint8Array, name: string): boolean;
+    plan(files: ImportFiles, io: BlobReader): Promise<ImportPlan>;
+    /** Batches of every selected group, in file order (groups may interleave in time). */
+    stream(args: StreamArgs): AsyncIterable<Batch>;
+    /** One group's batches in time order, reading only that group (formats that store groups apart). */
+    streamGroup?(args: StreamArgs, group: number): AsyncIterable<Batch>;
+}
+/** The channels selected in group `g` (empty when the group is not selected). */
+export declare function selectedChannels(args: StreamArgs, g: number): number[];
+/** What a binary dataset knows about its file (`LoadedDataset.info`, `DatasetInfo.fileInfo`). */
+export interface DatasetFileInfo {
+    format: string;
+    /** Importer id, used to read the file again (split / combine). */
+    importer: string;
+    startTime: string | null;
+    /** Every group in the file, also those not read. */
+    groups: {
+        name: string;
+        channels: number;
+        samples: number;
+        rateHz: number | null;
+    }[];
+    /** What this dataset holds. */
+    selection: GroupSelection[];
+    /** Several groups on one time base: how many, and the base rate. */
+    combined: {
+        groups: number;
+        rateHz: number | null;
+    } | null;
+    /** Datasets split from one file share this key. */
+    splitKey: string | null;
+    channels: number;
+    notes: string[];
+    meta: Record<string, string>;
+    events: FileEvent[];
+}
+/** What `chooseChannels` is asked before a large or ambiguous binary file is read. */
+export interface ChannelRequest {
+    file: string;
+    format: string;
+    groups: {
+        name: string;
+        rateHz: number | null;
+        samples: number;
+        channels: {
+            name: string;
+            unit: string;
+        }[];
+    }[];
+    /** What would be read without a choice. */
+    selection: GroupSelection[];
+    /** Values (rows × columns) a selection would load. */
+    estimate(selection: GroupSelection[]): number;
+    /** In-memory budget and what open datasets already use (above it, the file goes out of core). */
+    budget: number;
+    used: number;
+    notes: string[];
+}
+/** Resolves with the channels to read, or null to cancel this file. */
+export type ChannelChooser = (request: ChannelRequest) => Promise<GroupSelection[] | null>;

package/dist/types/lib/importers/workerCore.d.ts

@@ -0,0 +1,44 @@
+import { type ReadResult } from './run';
+import type { Chunk } from './combine';
+import type { GroupSelection, ImportFiles, ImportPlan, Importer } from './types';
+export type ToWorker = {
+    type: 'plan';
+    importer: string;
+    files: ImportFiles;
+} | {
+    type: 'read';
+    selection: GroupSelection[];
+    mode: 'memory' | 'stream';
+} | {
+    type: 'ack';
+}
+/** Load h5wasm now (warm-up ahead of an HDF5 / MAT 7.3 import). */
+ | {
+    type: 'warm';
+};
+export type FromWorker = {
+    type: 'plan';
+    plan: ImportPlan;
+} | {
+    type: 'progress';
+    fraction: number;
+} | {
+    type: 'chunk';
+    chunk: Chunk;
+} | {
+    type: 'done';
+    result: ReadResult;
+} | {
+    type: 'error';
+    message: string;
+}
+/** Heartbeat while a request runs; `work` (bytes read) shows whether it is getting anywhere. */
+ | {
+    type: 'alive';
+    work: number;
+};
+export declare const HEARTBEAT_MS = 1000;
+/** A chunk ready to post: arrays owning their buffer are transferred, views into a larger buffer copied. */
+export declare function transferable(chunk: Chunk): [Chunk, ArrayBuffer[]];
+/** The import worker: plan once, then any number of reads (a split reads each group). */
+export declare function workerHandler(post: (m: FromWorker, transfer?: Transferable[]) => void, find?: (id: string) => Importer): (m: ToWorker) => Promise<void>;

package/dist/types/lib/importers/xdf/format.d.ts

@@ -0,0 +1,2 @@
+import type { FormatInfo } from '../formats';
+export declare const XDF_FORMAT: FormatInfo;

package/dist/types/lib/importers/xdf/importer.d.ts

@@ -0,0 +1,2 @@
+import { type Importer } from '../types';
+export declare const xdfImporter: Importer;

package/dist/types/lib/importers/zlib.d.ts

@@ -0,0 +1,13 @@
+/**
+ * A zlib stream arriving in pieces, inflated as it comes (never the whole input
+ * or output at once): the platform's `DecompressionStream('deflate')` (browsers,
+ * workers, Node 18+), else fflate. Stopping early (break) cancels the inflate.
+ */
+export declare function inflateStream(input: AsyncIterable<Uint8Array>): AsyncGenerator<Uint8Array>;
+/**
+ * A whole zlib stream (MF4 DZ blocks, MAT miCOMPRESSED elements), by the
+ * platform's zlib when it has one. `size`: the inflated length when known (one
+ * output buffer, no concatenation). `limit`: stop once that many bytes are out;
+ * the result is then the first `limit` bytes.
+ */
+export declare function inflate(b: Uint8Array, size?: number, limit?: number): Promise<Uint8Array>;

package/dist/types/lib/kde.d.ts

@@ -0,0 +1,2 @@
+/** Gaussian KDE of `sorted` (any order) at `pts`; over many values, of their histogram on [lo, hi]. */
+export declare function kde(sorted: ArrayLike<number>, h: number, pts: Float64Array, lo: number, hi: number): Float64Array;