toy 0.8.0

data/tinynn/tinynn_gguf.h

@@ -0,0 +1,167 @@
+#ifndef TINYNN_GGUF_H
+#define TINYNN_GGUF_H
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Minimal GGUF loading wrapper around ggml's gguf_init_from_file —
+ * needed because Spinel's FFI can't pass `struct gguf_init_params` by
+ * value. The session owns both the gguf metadata context and the
+ * ggml context that holds tensor headers + data.
+ */
+void  *tnn_gguf_load(const char *path);            /* NULL on error */
+void  *tnn_gguf_load_empty(void);                   /* empty GGUF, 0 tensors */
+void   tnn_gguf_free(void *handle);
+
+int    tnn_gguf_n_tensors(void *handle);
+const char *tnn_gguf_tensor_name(void *handle, int i);
+int    tnn_gguf_tensor_ne(void *handle, int i, int dim);   /* dim 0..3 */
+int    tnn_gguf_tensor_type(void *handle, int i);           /* GGML_TYPE_F32, etc. */
+size_t tnn_gguf_tensor_nbytes(void *handle, int i);
+
+/* Copy tensor data into a caller-owned double buffer (f64).
+ * `n` is the number of elements (caller should query nbytes/elementsize).
+ * Currently supports GGML_TYPE_F32 only — future: dequantize Q4/Q8/etc.
+ */
+/* Read tensor data as doubles. Handles F32 directly and Q8_0 / Q4_K /
+ * Q5_K / Q6_K / F16 / BF16 / etc. via ggml's type_traits.to_float
+ * dequantization path. */
+int    tnn_gguf_read_f32_to_doubles(void *handle, int i, double *out, size_t n);
+int    tnn_gguf_tensor_is_quantized(void *handle, int i);
+
+/* Read scalar metadata kv pairs. The architecture decides the keys
+ * (e.g. "gpt2.embedding_length", "gpt2.attention.layer_norm_epsilon").
+ * Returns -1 / 0.0 if the key is missing or has the wrong type. */
+int    tnn_gguf_get_u32(void *handle, const char *key);
+double tnn_gguf_get_f32(void *handle, const char *key);
+int    tnn_gguf_get_bool(void *handle, const char *key);
+
+/* String metadata: returns NULL if key missing / not a string. The
+ * returned pointer is owned by ggml; do not free. */
+const char *tnn_gguf_get_str(void *handle, const char *key);
+
+/* Array metadata. n() returns -1 on missing / non-array. arr_str()
+ * returns NULL on bounds error or non-string-typed array. */
+int         tnn_gguf_arr_n  (void *handle, const char *key);
+const char *tnn_gguf_arr_str(void *handle, const char *key, int i);
+
+/* Phase 2 BYO-pointer mmap accessors. Used to wire weight tensors to
+ * point at file pages via tnn_session_attach_weight_mmap. */
+void  *tnn_gguf_mmap_base(void *handle);
+size_t tnn_gguf_mmap_size(void *handle);
+size_t tnn_gguf_tensor_file_offset(void *handle, int i);
+
+/* Create a tiny GGUF file at `path` with one 2x3 f32 tensor named
+ * "demo.tensor" containing [1.0, 2.0, ..., 6.0]. Useful for the
+ * smoke test to exercise the full load path without needing a
+ * downloaded model file. Returns 0 on success. */
+int    tnn_gguf_write_demo_file(const char *path);
+
+/* Direct GGUF → FFI persistent buffer loaders. Bypass the Ruby Mat /
+ * Array<Float64> intermediate so 7B-class models fit in RAM (3× win:
+ * 12 B/w → 4 B/w). The session arg is needed only when we have to
+ * stage a transpose; for direct memcpy the scratch buffer is unused.
+ *
+ * All four return 0 on success, negative on error (tensor missing,
+ * shape mismatch, dequant failure). The source tensor is dequantized
+ * to f32 on read if it isn't already; quantized GGUFs work but pay
+ * one dequant per call. */
+
+/* Single non-transposed tensor: token_embd.weight (V × D), and any
+ * full-shape weight without per-head splitting. ggml_backend_tensor_set
+ * handles large blits internally; no scratch chunking needed. */
+int tnn_gguf_copy_to_persistent(void *handle, int tensor_idx,
+                                 void *sess, void *target_tensor);
+
+/* Single 1-D tensor: output_norm.weight, attn_norm.weight, ffn_norm.weight. */
+int tnn_gguf_copy_1d_to_persistent(void *handle, int tensor_idx,
+                                    void *sess, void *target_tensor);
+
+/* Single transposed tensor: attn_output.weight, ffn_gate.weight,
+ * ffn_up.weight, ffn_down.weight. Source is (br × bc) row-major;
+ * target is the ggml ne=[br, bc] persistent tensor that expects the
+ * transposed byte order (matches stage_transposed_and_upload). Chunked
+ * by columns so it works for tensors larger than scratch.
+ *
+ * Convention reminder (same as tnn_upload_transposed_f64): the
+ * "transposition" here is just so that ggml_mul_mat(target, x) computes
+ * the mathematical (target^T · x) we want; bytes are physically
+ * reorganized once, at load time. */
+int tnn_gguf_copy_transposed_to_persistent(void *handle, int tensor_idx,
+                                            void *sess, void *target_tensor,
+                                            int br, int bc);
+
+/* Extract one head-slice from a full Q (n_heads × d_head) or K/V
+ * (n_kv × d_head) weight tensor and transpose-upload it to a single
+ * per-head persistent buffer. Source layout: row i is
+ * [head_0_col_0..d_head-1, head_1_col_0..d_head-1, ..., head_n-1_col_d_head-1]
+ * — i.e. heads are concatenated along the column axis. To get head h:
+ * src[i, h*d_head : (h+1)*d_head] for i in [0, d_model). Then
+ * transpose-write that (d_model × d_head) sub-matrix into target. */
+int tnn_gguf_copy_head_slice_to_persistent(void *handle, int tensor_idx,
+                                            void *sess, void *target_tensor,
+                                            int head_idx, int n_heads_total,
+                                            int d_model, int d_head);
+
+/* Native-layout per-head slice. Source has HF-native shape
+ * [n_heads_total*d_head, d_model] row-major (i.e. heads concatenated
+ * along the leading row axis). Head h is rows [h*d_head, (h+1)*d_head),
+ * which is a contiguous (d_head, d_model) row-major block whose bytes
+ * are already in ggml ne=[d_model, d_head] column-major order. Just a
+ * memcpy — no transpose, no chunking. Used by
+ * GGUFLoad.load_kv_cache_directly_native against GGUFs converted with
+ * --ggml-native. */
+int tnn_gguf_copy_head_slice_to_persistent_native(void *handle, int tensor_idx,
+                                                    void *sess, void *target_tensor,
+                                                    int head_idx, int n_heads_total,
+                                                    int d_model, int d_head);
+
+/* Verbatim (no-dequant) copy: bytes go from the GGUF tensor straight
+ * into the persistent ggml tensor. Caller must pre-allocate the
+ * destination with the same ggml type as the source. Used by Phase 3
+ * (Q8-stays-Q8). The dst type / size must match the GGUF tensor
+ * exactly; checked at runtime. */
+int tnn_gguf_copy_verbatim_to_persistent(void *handle, int tensor_idx,
+                                          void *sess, void *target_tensor);
+
+/* Per-head verbatim slice. Source must be native-layout (head h is
+ * rows [h*d_head, (h+1)*d_head) of an [n_heads_total*d_head, d_model]
+ * tensor). The destination is one per-head tensor; its ggml_nbytes
+ * must equal src_nbytes / n_heads_total. */
+int tnn_gguf_copy_verbatim_head_slice_to_persistent(void *handle, int tensor_idx,
+                                                      void *sess, void *target_tensor,
+                                                      int head_idx, int n_heads_total);
+
+/* Extract one head-slice from a 1-D bias tensor of length
+ * (n_heads_total × d_head) and copy it to target_tensor (1-D length
+ * d_head). Used for attn_q.bias / attn_k.bias / attn_v.bias in
+ * Qwen2.x models. */
+int tnn_gguf_copy_head_bias_slice_to_persistent(void *handle, int tensor_idx,
+                                                  void *sess, void *target_tensor,
+                                                  int head_idx, int d_head);
+
+/* Locate a tensor by name. Returns its index in the GGUF, or -1 if not
+ * found. Equivalent to the Ruby find_index helper but with no per-name
+ * linear-scan-in-Ruby overhead. */
+int tnn_gguf_find_index(void *handle, const char *name);
+
+/* Recursively scan `root` for *.gguf files. Returns a static buffer
+ * of newline-separated absolute paths (empty string if none found).
+ * Buffer is reused across calls — copy if needed. Used by
+ * lib/model_index.rb to find models in standard cache locations
+ * (HF hub, Ollama, LM Studio, ./models, ...). */
+const char *tnn_list_ggufs(const char *root);
+
+/* File size in bytes via stat(2). Returns 0 if path doesn't exist
+ * or stat fails (model files are never 0 bytes, so the ambiguity
+ * doesn't bite). */
+size_t tnn_file_size(const char *path);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/tinynn/tinynn_trace.c

@@ -0,0 +1,180 @@
+/* See tinynn_trace.h. */
+
+#include "tinynn_trace.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+/* The ring is fixed-size for predictable memory. 1048576 * 24 bytes =
+ * 24 MB. Sized for "one full native-Mat training run under trace"
+ * which is ~6 Mat-ops × ~100/step × ~hundreds of steps. FFI workloads
+ * use far fewer events; this just sets the ceiling. */
+#define TNN_TRACE_BUF_SIZE (1024 * 1024)
+
+typedef struct {
+    const char *name;
+    int64_t     ts_us;   /* offset from g_trace.epoch_us */
+    int64_t     dur_us;
+} tnn_trace_evt;
+
+static struct {
+    int             active;
+    FILE           *out;
+    tnn_trace_evt  *buf;
+    int             head;
+    int             wrap_warned;
+    int64_t         epoch_us;
+} g_trace = {0, NULL, NULL, 0, 0, 0};
+
+/* P6 per-op capture state. Flag is independent of g_trace.active so a
+ * caller can set it before tnn_trace_open and have it honoured once
+ * the trace opens. tnn_trace_op_capture_active() ANDs both. */
+static int     g_op_capture       = 0;
+static int64_t g_op_pending_start = 0;
+
+static int64_t now_us(void) {
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (int64_t)ts.tv_sec * 1000000 + (int64_t)(ts.tv_nsec / 1000);
+}
+
+int tnn_trace_active(void) {
+    return g_trace.active;
+}
+
+int tnn_trace_open(const char *path) {
+    if (g_trace.active) return -1;
+    if (!path) return -2;
+    g_trace.out = fopen(path, "w");
+    if (!g_trace.out) return -3;
+    g_trace.buf = (tnn_trace_evt *)calloc(TNN_TRACE_BUF_SIZE, sizeof(tnn_trace_evt));
+    if (!g_trace.buf) {
+        fclose(g_trace.out);
+        g_trace.out = NULL;
+        return -4;
+    }
+    /* Chrome Trace Format accepts either a top-level array or an
+     * object with a "traceEvents" field. We emit the array form — it
+     * streams cleanly with one record per line, and the close marker
+     * is just a single "]". */
+    fprintf(g_trace.out, "[\n");
+    g_trace.epoch_us    = now_us();
+    g_trace.head        = 0;
+    g_trace.wrap_warned = 0;
+    g_trace.active      = 1;
+    return 0;
+}
+
+int64_t tnn_trace_begin(const char *name) {
+    if (!g_trace.active) return 0;
+    (void)name;   /* unused at begin; we capture at end so a single
+                   * timestamp covers both push and pop. */
+    return now_us();
+}
+
+void tnn_trace_end(const char *name, int64_t start_ts) {
+    if (!g_trace.active) return;
+    int64_t end_ts = now_us();
+    int slot = g_trace.head;
+    if (slot >= TNN_TRACE_BUF_SIZE) {
+        if (!g_trace.wrap_warned) {
+            fprintf(stderr, "[tnn-trace] event buffer full at %d "
+                            "events; further events dropped.\n",
+                    TNN_TRACE_BUF_SIZE);
+            g_trace.wrap_warned = 1;
+        }
+        return;
+    }
+    g_trace.buf[slot].name   = name;
+    g_trace.buf[slot].ts_us  = start_ts - g_trace.epoch_us;
+    g_trace.buf[slot].dur_us = end_ts - start_ts;
+    g_trace.head = slot + 1;
+}
+
+void tnn_trace_set_op_capture(int en) {
+    g_op_capture = en ? 1 : 0;
+}
+
+int tnn_trace_op_capture_active(void) {
+    return g_op_capture && g_trace.active;
+}
+
+void tnn_trace_op_record_begin(void) {
+    if (!g_op_capture || !g_trace.active) return;
+    g_op_pending_start = now_us();
+}
+
+void tnn_trace_op_record_end(const char *op_name) {
+    if (!g_op_capture || !g_trace.active) return;
+    int slot = g_trace.head;
+    if (slot >= TNN_TRACE_BUF_SIZE) {
+        if (!g_trace.wrap_warned) {
+            fprintf(stderr, "[tnn-trace] event buffer full at %d "
+                            "events; further events dropped.\n",
+                    TNN_TRACE_BUF_SIZE);
+            g_trace.wrap_warned = 1;
+        }
+        return;
+    }
+    int64_t end_ts = now_us();
+    g_trace.buf[slot].name   = op_name;
+    g_trace.buf[slot].ts_us  = g_op_pending_start - g_trace.epoch_us;
+    g_trace.buf[slot].dur_us = end_ts - g_op_pending_start;
+    g_trace.head = slot + 1;
+}
+
+void tnn_trace_mark(const char *name) {
+    if (!g_trace.active) return;
+    int slot = g_trace.head;
+    if (slot >= TNN_TRACE_BUF_SIZE) return;   /* same overflow rule */
+    int64_t ts = now_us();
+    g_trace.buf[slot].name   = name;
+    g_trace.buf[slot].ts_us  = ts - g_trace.epoch_us;
+    g_trace.buf[slot].dur_us = 0;
+    g_trace.head = slot + 1;
+}
+
+/* Escape JSON string for output. Names should be plain ASCII (they're
+ * literals in our code) so the only escapes we expect are "\\" and
+ * "\"". We bail loud if we see a control char — that's a programmer
+ * error, not a runtime case. */
+static void write_escaped(FILE *f, const char *s) {
+    while (*s) {
+        unsigned char c = (unsigned char)*s++;
+        if (c == '"' || c == '\\') {
+            fputc('\\', f); fputc((int)c, f);
+        } else if (c < 0x20) {
+            fprintf(f, "\\u%04x", c);
+        } else {
+            fputc((int)c, f);
+        }
+    }
+}
+
+void tnn_trace_close(void) {
+    if (!g_trace.active) return;
+    g_trace.active = 0;   /* stop accepting events first; serialization
+                           * is unguarded. */
+    for (int i = 0; i < g_trace.head; i++) {
+        const tnn_trace_evt *e = &g_trace.buf[i];
+        fprintf(g_trace.out, "%s {", i == 0 ? " " : ",");
+        if (e->dur_us > 0) {
+            fprintf(g_trace.out, "\"ph\":\"X\",\"ts\":%ld,\"dur\":%ld,\"name\":\"",
+                    (long)e->ts_us, (long)e->dur_us);
+        } else {
+            /* instant event */
+            fprintf(g_trace.out, "\"ph\":\"i\",\"ts\":%ld,\"name\":\"",
+                    (long)e->ts_us);
+        }
+        write_escaped(g_trace.out, e->name ? e->name : "(null)");
+        fprintf(g_trace.out, "\",\"pid\":1,\"tid\":1}\n");
+    }
+    fprintf(g_trace.out, "]\n");
+    fclose(g_trace.out);
+    free(g_trace.buf);
+    g_trace.out = NULL;
+    g_trace.buf = NULL;
+    g_trace.head = 0;
+}

data/tinynn/tinynn_trace.h

@@ -0,0 +1,85 @@
+/* Chrome Trace Format emitter.
+ *
+ * One global trace target at a time. Open → instrument → close. The
+ * "off" state is fast: each tnn_trace_begin/end checks a single static
+ * int and returns immediately. We measured ~5ns/pair when off, dwarfed
+ * by anything else in the inner loop.
+ *
+ * Storage is a fixed-size ring of events; when full, further events
+ * are dropped with a one-shot stderr warning. Default buffer is 65536
+ * events, which at v1 instrumentation (FFI boundaries + Ruby step
+ * loop) is enough for ~1000 training steps.
+ *
+ * Output is a Chrome Trace JSON array on close — open in
+ * https://perfetto.dev or chrome://tracing.
+ *
+ * IMPORTANT: every `name` passed to tnn_trace_begin/end must be a
+ * string with stable address (string literal, or pointer to long-
+ * lived storage). We do NOT copy; flush time re-reads the pointer.
+ */
+
+#ifndef TINYNN_TRACE_H
+#define TINYNN_TRACE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+
+/* Open a trace file. Returns 0 on success, negative on failure.
+ * Idempotent calls when already-open return -1. */
+int     tnn_trace_open(const char *path);
+
+/* Push a region; returns a token (the start timestamp). Pair with
+ * tnn_trace_end. When tracing is off, returns 0 and does no work. */
+int64_t tnn_trace_begin(const char *name);
+
+/* Pop a region. `name` and `start_ts` must match the
+ * tnn_trace_begin call. When tracing is off, does nothing. */
+void    tnn_trace_end(const char *name, int64_t start_ts);
+
+/* One-shot instant event (no duration). Useful for markers. */
+void    tnn_trace_mark(const char *name);
+
+/* Flush events and close the file. */
+void    tnn_trace_close(void);
+
+/* Cheap predicate (one load) — callers can guard expensive
+ * instrumentation (e.g. snprintf'd dynamic names) so the OFF path
+ * is truly zero-cost. */
+int     tnn_trace_active(void);
+
+/* Per-op timing (P6). Opt-in: enabled by tnn_trace_set_op_capture(1)
+ * AFTER tnn_trace_open. When enabled, ggml-backend's sched eval
+ * callback (installed by tinynn_ggml.c at engine init) routes each
+ * node through tnn_trace_op_record_{begin,end}, which emit one
+ * Chrome-Trace duration event per ggml op (name = ggml_op_name(t->op)).
+ *
+ * Cost when OFF (capture flag = 0): one load + compare per ggml node.
+ *   Measured: identical wall time to a build without the eval
+ *   callback installed (within timing noise).
+ * Cost when ON: ~5× slowdown of compute_backward at SmolLM2-135M
+ *   LoRA shape on CPU. Most of the overhead is not the timestamp /
+ *   ring-buffer writes (cheap) but ggml-backend's per-node
+ *   callback dispatch path, which disables certain kernel fusion
+ *   opportunities. Acceptable for diagnostic use; do NOT leave on
+ *   for production profiling of throughput.
+ *
+ * Caveat (CUDA): eval callback fires after CPU enqueue, not after
+ * kernel completion. CUDA per-op timings reflect launch latency
+ * rather than kernel duration. CPU per-op timings are wall-accurate.
+ *
+ * Names are ggml_op_name(...) returns, which point into ggml's
+ * static const table — stable for the trace lifetime as required by
+ * the buffer contract above. */
+void    tnn_trace_set_op_capture(int en);
+int     tnn_trace_op_capture_active(void);
+void    tnn_trace_op_record_begin(void);
+void    tnn_trace_op_record_end(const char *op_name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif