toy 0.8.0

data/vendor/ggml/examples/mnist/mnist-eval.cpp

@@ -0,0 +1,67 @@
+#include "ggml.h"
+#include "ggml-opt.h"
+
+#include "mnist-common.h"
+
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <ctime>
+#include <string>
+#include <thread>
+#include <vector>
+
+#if defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+int main(int argc, char ** argv) {
+    srand(time(NULL));
+    ggml_time_init();
+
+    if (argc != 4 && argc != 5) {
+        fprintf(stderr, "Usage: %s mnist-fc-f32.gguf data/MNIST/raw/t10k-images-idx3-ubyte data/MNIST/raw/t10k-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
+        exit(1);
+    }
+
+    ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTEST, MNIST_NBATCH_PHYSICAL);
+
+    if (!mnist_image_load(argv[2], dataset)) {
+        return 1;
+    }
+    if (!mnist_label_load(argv[3], dataset)) {
+        return 1;
+    }
+
+    const int iex = rand() % MNIST_NTEST;
+    mnist_image_print(stdout, dataset, iex);
+
+    const std::string backend = argc >= 5 ? argv[4] : "";
+
+    const int64_t t_start_us = ggml_time_us();
+    mnist_model model = mnist_model_init_from_file(argv[1], backend, MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
+    mnist_model_build(model);
+    const int64_t t_load_us = ggml_time_us() - t_start_us;
+    fprintf(stdout, "%s: loaded model in %.2lf ms\n", __func__, t_load_us / 1000.0);
+
+    ggml_opt_result_t result_eval = mnist_model_eval(model, dataset);
+
+    std::vector<int32_t> pred(MNIST_NTEST);
+    ggml_opt_result_pred(result_eval, pred.data());
+    fprintf(stdout, "%s: predicted digit is %d\n", __func__, pred[iex]);
+
+    double loss;
+    double loss_unc;
+    ggml_opt_result_loss(result_eval, &loss, &loss_unc);
+    fprintf(stdout, "%s: test_loss=%.6lf+-%.6lf\n", __func__, loss, loss_unc);
+
+    double accuracy;
+    double accuracy_unc;
+    ggml_opt_result_accuracy(result_eval, &accuracy, &accuracy_unc);
+    fprintf(stdout, "%s: test_acc=%.2lf+-%.2lf%%\n", __func__, 100.0*accuracy, 100.0*accuracy_unc);
+
+    ggml_opt_result_free(result_eval);
+
+    return 0;
+}

data/vendor/ggml/examples/mnist/mnist-train-cnn.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import sys
+from time import time
+import gguf
+import numpy as np
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras import layers
+
+
+def train(model_path):
+    # Model / data parameters
+    num_classes = 10
+    input_shape = (28, 28, 1)
+
+    # Load the data and split it between train and test sets
+    (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
+
+    # Scale images to the [0, 1] range
+    x_train = x_train.astype("float32") / 255
+    x_test = x_test.astype("float32") / 255
+    x_train = np.expand_dims(x_train, -1)
+    x_test = np.expand_dims(x_test, -1)
+    print("x_train shape:", x_train.shape)
+    print(x_train.shape[0], "train samples")
+    print(x_test.shape[0], "test samples")
+
+    # convert class vectors to binary class matrices
+    y_train = keras.utils.to_categorical(y_train, num_classes)
+    y_test = keras.utils.to_categorical(y_test, num_classes)
+
+    model = keras.Sequential(
+        [
+            keras.Input(shape=input_shape, dtype=tf.float32),
+            layers.Conv2D(8, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
+            layers.MaxPooling2D(pool_size=(2, 2)),
+            layers.Conv2D(16, kernel_size=(3, 3), padding="same", activation="relu", dtype=tf.float32),
+            layers.MaxPooling2D(pool_size=(2, 2)),
+            layers.Flatten(),
+            layers.Dense(num_classes, activation="softmax", dtype=tf.float32),
+        ]
+    )
+
+    model.summary()
+    batch_size = 1000
+    epochs = 30
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
+
+    t_start = time()
+    model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
+    print(f"Training took {time()-t_start:.2f}s")
+
+    score = model.evaluate(x_test, y_test, verbose=0)
+    print(f"Test loss: {score[0]:.6f}")
+    print(f"Test accuracy: {100*score[1]:.2f}%")
+
+    gguf_writer = gguf.GGUFWriter(model_path, "mnist-cnn")
+
+    conv1_kernel = model.layers[0].weights[0].numpy()
+    conv1_kernel = np.moveaxis(conv1_kernel, [2, 3], [0, 1])
+    gguf_writer.add_tensor("conv1.kernel", conv1_kernel, raw_shape=(8, 1, 3, 3))
+
+    conv1_bias = model.layers[0].weights[1].numpy()
+    gguf_writer.add_tensor("conv1.bias", conv1_bias, raw_shape=(1, 8, 1, 1))
+
+    conv2_kernel = model.layers[2].weights[0].numpy()
+    conv2_kernel = np.moveaxis(conv2_kernel, [0, 1, 2, 3], [2, 3, 1, 0])
+    gguf_writer.add_tensor("conv2.kernel", conv2_kernel, raw_shape=(16, 8, 3, 3))
+
+    conv2_bias = model.layers[2].weights[1].numpy()
+    gguf_writer.add_tensor("conv2.bias", conv2_bias, raw_shape=(1, 16, 1, 1))
+
+    dense_weight = model.layers[-1].weights[0].numpy()
+    dense_weight = dense_weight.transpose()
+    gguf_writer.add_tensor("dense.weight", dense_weight, raw_shape=(10, 7*7*16))
+
+    dense_bias = model.layers[-1].weights[1].numpy()
+    gguf_writer.add_tensor("dense.bias", dense_bias)
+
+    gguf_writer.write_header_to_file()
+    gguf_writer.write_kv_data_to_file()
+    gguf_writer.write_tensors_to_file()
+    gguf_writer.close()
+    print(f"GGUF model saved to '{model_path}'")
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <model_path>")
+        sys.exit(1)
+    train(sys.argv[1])

data/vendor/ggml/examples/mnist/mnist-train-fc.py

@@ -0,0 +1,131 @@
+import gguf
+import numpy as np
+import torch
+import torch.nn as nn
+import torchvision.datasets as dsets
+import torchvision.transforms as transforms
+from torch.autograd import Variable
+
+import sys
+from time import time
+
+input_size  = 784  # img_size = (28,28) ---> 28*28=784 in total
+hidden_size = 500  # number of nodes at hidden layer
+num_classes = 10   # number of output classes discrete range [0,9]
+num_epochs  = 30   # number of times which the entire dataset is passed throughout the model
+batch_size  = 1000 # the size of input data used for one iteration
+lr          = 1e-3 # size of step
+
+
+class Net(nn.Module):
+    def __init__(self, input_size, hidden_size, num_classes):
+        super(Net, self).__init__()
+        self.fc1 = nn.Linear(input_size, hidden_size)
+        self.relu = nn.ReLU()
+        self.fc2 = nn.Linear(hidden_size, num_classes)
+
+    def forward(self, x):
+        out = self.fc1(x)
+        out = self.relu(out)
+        out = self.fc2(out)
+        return out
+
+
+def train(model_path):
+    train_data = dsets.MNIST(root='./data', train=True, transform=transforms.ToTensor(), download=True)
+    test_data  = dsets.MNIST(root='./data', train=False, transform=transforms.ToTensor())
+
+    assert len(train_data) == 60000
+    assert len(test_data)  == 10000
+
+    kwargs_train_test = dict(batch_size=batch_size, num_workers=4, pin_memory=True)
+    train_gen = torch.utils.data.DataLoader(dataset=train_data, shuffle=True,  **kwargs_train_test)
+    test_gen  = torch.utils.data.DataLoader(dataset=test_data,  shuffle=False, **kwargs_train_test)
+
+    net = Net(input_size, hidden_size, num_classes)
+
+    if torch.cuda.is_available():
+        net.cuda()
+
+    loss_function = nn.CrossEntropyLoss()
+    optimizer = torch.optim.Adam(net.parameters(), lr=lr)
+
+    t_start = time()
+    for epoch in range(num_epochs):
+        loss_history = []
+        ncorrect = 0
+
+        for i, (images, labels) in enumerate(train_gen):
+            images = Variable(images.view(-1, 28*28))
+            labels = Variable(labels)
+
+            if torch.cuda.is_available():
+                images = images.cuda()
+                labels = labels.cuda()
+
+            optimizer.zero_grad()
+            outputs = net(images)
+            loss = loss_function(outputs, labels)
+
+            loss_history.append(loss.cpu().data)
+            _, predictions = torch.max(outputs, 1)
+            ncorrect += (predictions == labels).sum()
+
+            loss.backward()
+            optimizer.step()
+
+            if (i + 1)*batch_size % 10000 == 0:
+                loss_mean = np.mean(loss_history)
+                accuracy = ncorrect / ((i + 1) * batch_size)
+                print(
+                    f"Epoch [{epoch+1:02d}/{num_epochs}], "
+                    f"Step [{(i+1)*batch_size:05d}/{len(train_data)}], "
+                    f"Loss: {loss_mean:.4f}, Accuracy: {100*accuracy:.2f}%")
+    print()
+    print(f"Training took {time()-t_start:.2f}s")
+
+    loss_history = []
+    ncorrect = 0
+
+    for i, (images, labels) in enumerate(test_gen):
+        images = Variable(images.view(-1, 28*28))
+        labels = Variable(labels)
+
+        if torch.cuda.is_available():
+            images = images.cuda()
+            labels = labels.cuda()
+
+        outputs = net(images)
+        loss = loss_function(outputs, labels)
+
+        loss_history.append(loss.cpu().data)
+        _, predictions = torch.max(outputs, 1)
+        ncorrect += (predictions == labels).sum().cpu().numpy()
+
+    loss_mean            = np.mean(loss_history)
+    loss_uncertainty     = np.std(loss_history) / np.sqrt(len(loss_history) - 1)
+    accuracy_mean        = ncorrect / (len(test_gen) * batch_size)
+    accuracy_uncertainty = np.sqrt(accuracy_mean * (1.0 - accuracy_mean) / (len(test_gen) * batch_size))
+    print()
+    print(f"Test loss: {loss_mean:.6f}+-{loss_uncertainty:.6f}, Test accuracy: {100*accuracy_mean:.2f}+-{100*accuracy_uncertainty:.2f}%")
+
+    gguf_writer = gguf.GGUFWriter(model_path, "mnist-fc")
+
+    print()
+    print(f"Model tensors saved to {model_path}:")
+    for tensor_name in net.state_dict().keys():
+        data = net.state_dict()[tensor_name].squeeze().cpu().numpy()
+        print(tensor_name, "\t", data.shape)
+        gguf_writer.add_tensor(tensor_name, data)
+
+    gguf_writer.write_header_to_file()
+    gguf_writer.write_kv_data_to_file()
+    gguf_writer.write_tensors_to_file()
+    gguf_writer.close()
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} <model_path>")
+        sys.exit(1)
+    train(sys.argv[1])

data/vendor/ggml/examples/mnist/mnist-train.cpp

@@ -0,0 +1,39 @@
+#include "ggml-opt.h"
+#include "mnist-common.h"
+
+#include <cmath>
+#include <cstdio>
+#include <cstring>
+#include <ctime>
+#include <string>
+
+#if defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
+#endif
+
+int main(int argc, char ** argv) {
+    if (argc != 5 && argc != 6) {
+        fprintf(stderr, "Usage: %s mnist-fc mnist-fc-f32.gguf data/MNIST/raw/train-images-idx3-ubyte data/MNIST/raw/train-labels-idx1-ubyte [CPU/CUDA0]\n", argv[0]);
+        exit(0);
+    }
+
+    // The MNIST model is so small that the overhead from data shuffling is non-negligible, especially with CUDA.
+    // With a shard size of 10 this overhead is greatly reduced at the cost of less shuffling (does not seem to have a significant impact).
+    // A batch of 500 images then consists of 50 random shards of size 10 instead of 500 random shards of size 1.
+    ggml_opt_dataset_t dataset = ggml_opt_dataset_init(GGML_TYPE_F32, GGML_TYPE_F32, MNIST_NINPUT, MNIST_NCLASSES, MNIST_NTRAIN, /*ndata_shard =*/ 10);
+
+    if (!mnist_image_load(argv[3], dataset)) {
+        return 1;
+    }
+    if (!mnist_label_load(argv[4], dataset)) {
+        return 1;
+    }
+
+    mnist_model model = mnist_model_init_random(argv[1], argc >= 6 ? argv[5] : "", MNIST_NBATCH_LOGICAL, MNIST_NBATCH_PHYSICAL);
+
+    mnist_model_build(model);
+
+    mnist_model_train(model, dataset, /*nepoch =*/ 30, /*val_split =*/ 0.05f);
+
+    mnist_model_save(model, argv[2]);
+}

data/vendor/ggml/examples/mnist/server.py

@@ -0,0 +1,36 @@
+import http.server
+import socketserver
+import os
+import sys
+
+DIRECTORY = os.path.abspath(os.path.join(os.path.dirname(__file__), 'web'))
+PORT = 8000
+
+class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, directory=DIRECTORY, **kwargs)
+
+    def end_headers(self):
+        # Add required headers for SharedArrayBuffer
+        self.send_header("Cross-Origin-Opener-Policy", "same-origin")
+        self.send_header("Cross-Origin-Embedder-Policy", "require-corp")
+        self.send_header("Access-Control-Allow-Origin", "*")
+        super().end_headers()
+
+# Enable address reuse
+class CustomServer(socketserver.TCPServer):
+    allow_reuse_address = True
+
+try:
+    with CustomServer(("", PORT), CustomHTTPRequestHandler) as httpd:
+        print(f"Serving directory '{DIRECTORY}' at http://localhost:{PORT}")
+        print(f"Application context root: http://localhost:{PORT}/")
+        try:
+            httpd.serve_forever()
+        except KeyboardInterrupt:
+            print("\nServer stopped.")
+            # Force complete exit
+            sys.exit(0)
+except OSError as e:
+    print(f"Error: {e}")
+    sys.exit(1)

data/vendor/ggml/examples/mnist/web/index.html

@@ -0,0 +1,178 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>MNIST with GGML</title>
+    <script src="mnist.js"></script>
+</head>
+<body>
+    <h2>MNIST digit recognizer with <a href="https://github.com/ggerganov/ggml">GGML</a></h2>
+    <p id="msg">Loading model and data set, please wait ...</p>
+    <canvas id="ggCanvas" width="364" height="364" style="border:2px solid #d3d3d3;">
+        Your browser does not support the HTML canvas tag.
+    </canvas>
+    <div>
+        <button id="clear" onclick="onClear()">Clear</button>
+        <button id="random" onclick="onRandom()" disabled>Random</button>
+        <button id="download" onclick="onDownload()">Download</button>
+    </div>
+    <div>
+        <p id="prediction"></p>
+    </div>
+    <script>
+"use strict";
+const DIGIT_SIZE = 28; // digits are 28x28 pixels
+var canvas = document.getElementById("ggCanvas");
+var ctx = canvas.getContext("2d", { alpha: false, willReadFrequently: true });
+ctx.fillStyle = "white";
+ctx.fillRect(0, 0, canvas.width, canvas.height);
+var dragging = false;
+var lastX, lastY;
+
+function onClear(event) {
+    ctx.fillStyle = "white";
+    ctx.fillRect(0, 0, canvas.width, canvas.height);
+    document.getElementById("prediction").innerHTML = "";
+}
+
+function predict(digit) {
+    let buf = Module._malloc(digit.length);
+    if (buf == 0) {
+        console.log("failed to allocate memory");
+        return;
+    }
+    Module.HEAPU8.set(digit, buf);
+    let prediction = Module.ccall('wasm_eval', null, ['number'], [buf]);
+    Module._free(buf);
+    if (prediction >= 0) {
+        document.getElementById("prediction").innerHTML = "Predicted digit is <b>" + prediction + "</b>";
+    }
+}
+
+function onRandom(event) {
+    onClear();
+    const bufLength = DIGIT_SIZE*DIGIT_SIZE;
+    var buf = Module._malloc(bufLength);
+    if (buf == 0) {
+        console.log("failed to allocate memory");
+        return;
+    }
+    let ret = Module.ccall('wasm_random_digit', null, ['number'], [buf]);
+    let digit = new Uint8Array(Module.HEAPU8.buffer, buf, bufLength);
+    for (let i = 0; i < digit.length; i++) {
+        let x = i % DIGIT_SIZE;
+        let y = Math.floor(i / DIGIT_SIZE);
+        setPixel(x, y, digit[i]);
+    }
+    Module._free(buf);
+    predict(digit);
+}
+
+function onDownload(event) {
+    let digit = scaleCanvas();
+    let digitBlob = new Blob([new Uint8Array(digit)], {type: "application/octet-stream"});
+    let url = URL.createObjectURL(digitBlob);
+    let link = document.createElement('a');
+    link.href = url;
+    link.download = "image.raw";
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+}
+
+// Get the position of the mouse relative to the canvas
+function getMousePos(event) {
+    if (event.touches !== undefined && event.touches.length > 0) {
+        event = event.touches[0];
+    }
+    var rect = canvas.getBoundingClientRect();
+    return [Math.floor(event.clientX) - rect.left, Math.floor(event.clientY) - rect.top];
+}
+
+function setPixel(x, y, val) {
+    let canvasX = x * 13;
+    let canvasY = y * 13;
+    let color = 255 - val;
+    ctx.fillStyle = "#" + color.toString(16) + color.toString(16) + color.toString(16);
+    ctx.fillRect(canvasX, canvasY, 13, 13);
+}
+
+function onMouseDown(e) {
+    dragging = true;
+    [lastX, lastY] = getMousePos(e);
+}
+
+// scale the canvas to 28x28 pixels and return the pixel values as an array
+function scaleCanvas() {
+    let imgData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+    let tempCanvas = document.createElement('canvas');
+    tempCanvas.width = DIGIT_SIZE;
+    tempCanvas.height = DIGIT_SIZE;
+    let tempCtx = tempCanvas.getContext("2d");
+    tempCtx.drawImage(canvas, 0, 0, DIGIT_SIZE, DIGIT_SIZE);
+    let tempImgData = tempCtx.getImageData(0, 0, DIGIT_SIZE, DIGIT_SIZE);
+    let tempData = tempImgData.data;
+    let digit = new Array(DIGIT_SIZE*DIGIT_SIZE).fill(0);
+    for (let i = 0; i < tempData.length; i += 4) {
+        let val = 255 - tempData[i];
+        digit[i / 4] = val;
+    }
+    return digit;
+}
+
+function onMouseUp(e) {
+    dragging = false;
+    let digit = scaleCanvas();
+    predict(digit);
+}
+
+function onMouseMove(e) {
+    if (dragging) {
+        let [mouseX, mouseY] = getMousePos(e);
+        ctx.beginPath();
+        ctx.moveTo(lastX, lastY);
+        ctx.lineTo(mouseX, mouseY);
+        ctx.lineWidth = 20;
+        ctx.lineJoin = ctx.lineCap = 'round';
+        ctx.strokeStyle = "#000000";
+        ctx.stroke();
+        ctx.closePath();
+        lastX = mouseX;
+        lastY = mouseY;
+    }
+}
+
+// Prevent scrolling when touching the canvas
+document.body.addEventListener("touchstart", function (e) {
+if (e.target == canvas) {
+    e.preventDefault();
+}
+}, {passive: false});
+document.body.addEventListener("touchend", function (e) {
+if (e.target == canvas) {
+    e.preventDefault();
+}
+}, {passive: false});
+document.body.addEventListener("touchmove", function (e) {
+if (e.target == canvas) {
+    e.preventDefault();
+}
+}, {passive: false});
+
+function onRuntimeInitialized() {
+    // Use the same handlers for mouse and touch events
+    canvas.onmousedown = onMouseDown;
+    canvas.onmouseup = onMouseUp;
+    canvas.onmousemove = onMouseMove;
+    canvas.ontouchstart = onMouseDown;
+    canvas.ontouchend = onMouseUp;
+    canvas.ontouchmove = onMouseMove;
+    document.getElementById("msg").innerHTML = "Draw a single digit on the canvas below:"
+    document.getElementById("random").disabled = false;
+}
+
+Module['onRuntimeInitialized'] = onRuntimeInitialized;
+    </script>
+</body>
+</html>

data/vendor/ggml/examples/perf-metal/CMakeLists.txt

@@ -0,0 +1,7 @@
+#
+# perf-metal
+
+set(TEST_TARGET perf-metal)
+add_executable(${TEST_TARGET} perf-metal.cpp)
+target_link_libraries(${TEST_TARGET} PRIVATE ggml)
+