@tangle-network/starter-foundry 0.5.4 → 0.7.0

package/registry/families/tgi-server/manifest.json

@@ -0,0 +1,178 @@
+{
+  "id": "tgi-server",
+  "description": "HuggingFace Text Generation Inference server. Docker-compose-first LLM serving with HF's production Rust/Python server — continuous batching, Flash Attention 2, quantization (AWQ, GPTQ, EETQ, bitsandbytes), tensor-parallel sharding, and a native /generate + OpenAI-compatible surface. Pulls weights straight from the Hub.",
+  "tags": [
+    "ml",
+    "llm",
+    "inference",
+    "python",
+    "serving",
+    "huggingface",
+    "tgi"
+  ],
+  "taxonomy": {
+    "language": "python",
+    "runtime": "python",
+    "surface": "inference"
+  },
+  "slots": {
+    "auth": {
+      "options": [
+        "auth:none",
+        "auth:better-auth",
+        "auth:clerk"
+      ],
+      "default": "auth:none"
+    }
+  },
+  "defaults": {
+    "projectType": "inference",
+    "serviceName": "starter-foundry-tgi",
+    "port": 8080,
+    "modelId": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "maxInputLength": 3072,
+    "maxTotalTokens": 4096
+  },
+  "files": [
+    {
+      "source": "files/docker-compose.yml",
+      "target": "docker-compose.yml"
+    },
+    {
+      "source": "files/requirements.txt",
+      "target": "requirements.txt"
+    },
+    {
+      "source": "files/client.py",
+      "target": "src/client.py"
+    },
+    {
+      "source": "files/.env.example",
+      "target": ".env.example"
+    },
+    {
+      "source": "files/README-tgi.md",
+      "target": "docs/tgi.md"
+    },
+    {
+      "source": "files/validate-tgi.mjs",
+      "target": "validate-tgi.mjs"
+    }
+  ],
+  "validationChecks": [
+    {
+      "type": "file-exists",
+      "path": "docker-compose.yml"
+    },
+    {
+      "type": "file-exists",
+      "path": "requirements.txt"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/client.py"
+    },
+    {
+      "type": "file-exists",
+      "path": ".env.example"
+    },
+    {
+      "type": "command-success",
+      "command": [
+        "node",
+        "validate-tgi.mjs"
+      ],
+      "expect": "tgi starter ok"
+    }
+  ],
+  "contextHints": {
+    "commands": [
+      "cp .env.example .env",
+      "docker compose up -d",
+      "pip install -r requirements.txt",
+      "python -m src.client"
+    ],
+    "entrypoints": [
+      "docker-compose.yml",
+      "src/client.py",
+      ".env.example"
+    ],
+    "preview": {
+      "path": "/health",
+      "port": "{{port}}"
+    },
+    "extensionPoints": [
+      "docker-compose.yml",
+      "src/client.py"
+    ]
+  },
+  "keywords": [
+    "tgi",
+    "text generation inference",
+    "huggingface inference",
+    "hf tgi",
+    "hf inference server",
+    "ghcr tgi",
+    "flash attention serving",
+    "hub model serving",
+    "docker compose llm",
+    "generate endpoint"
+  ],
+  "scoring": {
+    "boost": {
+      "tgi": 4,
+      "text generation inference": 4,
+      "huggingface inference": 3
+    }
+  },
+  "tieredKeywords": {
+    "tier1": [
+      "tgi",
+      "text generation inference",
+      "huggingface inference server",
+      "hf tgi"
+    ],
+    "tier2": [
+      "flash attention serving",
+      "hub model serving",
+      "/generate endpoint",
+      "gptq awq eetq serving"
+    ],
+    "archetypes": [
+      "huggingface self-hosted llm",
+      "dockerized tgi deployment",
+      "hub-pull inference"
+    ]
+  },
+  "buildHints": {
+    "whenToUse": "Pick TGI when the model lives on the HuggingFace Hub, docker-compose is the preferred deploy surface, and you want HF's production server (continuous batching, Flash Attention, multi-GPU sharding) without writing a FastAPI wrapper. Choose vllm-server instead when you need PagedAttention / custom OpenAI-compat Python glue; choose ollama-server for local GGUF-first workflows.",
+    "firstSteps": [
+      "Copy .env.example to .env and set HUGGING_FACE_HUB_TOKEN (required for gated models: Llama, Mistral, Gemma)",
+      "Edit MODEL_ID in .env or docker-compose.yml to the HF repo you want to serve",
+      "docker compose up -d — first boot pulls weights to the ./data volume (can be 15-60GB)",
+      "Call `curl localhost:{{port}}/generate -d '{\"inputs\":\"hello\"}'` or run `python -m src.client`"
+    ],
+    "gotchas": [
+      "TGI pins a specific transformers version per image tag. Use a fixed tag (e.g. 2.3.1) in production — `latest` can silently break custom architectures.",
+      "Gated models (Llama, Mistral, Gemma) require HUGGING_FACE_HUB_TOKEN AND acceptance of the model license on the Hub UI — token alone is not enough.",
+      "SHM size: the default docker /dev/shm is 64MB which crashes NCCL on multi-GPU. Keep the `shm_size: 1gb` block in docker-compose.yml.",
+      "MAX_INPUT_LENGTH must be strictly less than MAX_TOTAL_TOKENS (the difference is the generation budget). TGI refuses to start if they're equal.",
+      "Quantization: `--quantize awq` needs AWQ-packed weights; `bitsandbytes-nf4` works on raw weights but is slower. Don't mix.",
+      "CUDA: requires the nvidia container toolkit. `docker compose up` with no GPU silently falls back to CPU and OOMs on any non-tiny model."
+    ],
+    "placeholders": [
+      {
+        "path": "src/main.py",
+        "description": ".env (MODEL_ID, HUGGING_FACE_HUB_TOKEN)"
+      },
+      {
+        "path": "src/main.py",
+        "description": "docker-compose.yml (image tag pin, GPU device count, volume path)"
+      },
+      {
+        "path": "src/main.py",
+        "description": "src/client.py (prompt + generation parameters)"
+      }
+    ]
+  }
+}

package/registry/families/threejs-game/files/index.html

@@ -0,0 +1,16 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>{{serviceName}}</title>
+    <style>
+      html, body { margin: 0; padding: 0; height: 100%; background: #0b0d10; overflow: hidden; }
+      canvas { display: block; width: 100vw; height: 100vh; }
+    </style>
+  </head>
+  <body>
+    <canvas id="scene"></canvas>
+    <script type="module" src="/src/main.ts"></script>
+  </body>
+</html>

package/registry/families/threejs-game/files/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "{{packageName}}",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview",
+    "validate": "node validate-threejs.mjs"
+  },
+  "dependencies": {
+    "three": "^0.170.0"
+  },
+  "devDependencies": {
+    "@types/three": "^0.170.0",
+    "typescript": "^5.9.0",
+    "vite": "^6.4.2"
+  },
+  "overrides": {
+    "vite": "^6.4.2"
+  },
+  "pnpm": {
+    "overrides": {
+      "vite": "^6.4.2"
+    }
+  }
+}

package/registry/families/threejs-game/files/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "noEmit": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"]
+  },
+  "include": ["src", "vite.config.ts"]
+}

package/registry/families/threejs-game/files/validate-threejs.mjs

@@ -0,0 +1,32 @@
+import fs from "node:fs/promises";
+
+const [pkgSource, mainSource, htmlSource] = await Promise.all([
+  fs.readFile("package.json", "utf8"),
+  fs.readFile("src/main.ts", "utf8"),
+  fs.readFile("index.html", "utf8"),
+]);
+
+const pkg = JSON.parse(pkgSource);
+if (pkg.name !== "{{packageName}}") {
+  throw new Error(`package.json name should be "{{packageName}}", got "${pkg.name}"`);
+}
+
+const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
+if (!deps.three) throw new Error("package.json missing `three` dependency");
+if (!deps.vite) throw new Error("package.json missing `vite` devDependency");
+
+if (!mainSource.includes("WebGLRenderer")) {
+  throw new Error("src/main.ts should construct a THREE.WebGLRenderer");
+}
+if (!mainSource.includes("requestAnimationFrame")) {
+  throw new Error("src/main.ts should drive a requestAnimationFrame loop");
+}
+if (!mainSource.includes("resize")) {
+  throw new Error("src/main.ts should register a resize handler");
+}
+
+if (!htmlSource.includes('src="/src/main.ts"')) {
+  throw new Error("index.html should load /src/main.ts as a module");
+}
+
+console.log("threejs game starter ok");

package/registry/families/threejs-game/files/vite.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vite'
+
+export default defineConfig({
+  server: {
+    port: 5173,
+  },
+  build: {
+    target: 'es2022',
+  },
+})

package/registry/families/threejs-game/manifest.json

@@ -0,0 +1,164 @@
+{
+  "id": "threejs-game",
+  "description": "Three.js WebGL 3D scene starter on Vite with a renderer, camera, lit mesh, and resize loop wired from first render.",
+  "tags": [
+    "frontend",
+    "typescript",
+    "three.js",
+    "webgl",
+    "game",
+    "3d"
+  ],
+  "taxonomy": {
+    "language": "typescript",
+    "runtime": "node",
+    "surface": "frontend"
+  },
+  "defaults": {
+    "projectType": "frontend",
+    "serviceName": "starter-foundry-threejs",
+    "port": 5173
+  },
+  "files": [
+    {
+      "source": "files/package.json",
+      "target": "package.json"
+    },
+    {
+      "source": "files/tsconfig.json",
+      "target": "tsconfig.json"
+    },
+    {
+      "source": "files/vite.config.ts",
+      "target": "vite.config.ts"
+    },
+    {
+      "source": "files/index.html",
+      "target": "index.html"
+    },
+    {
+      "source": "files/validate-threejs.mjs",
+      "target": "validate-threejs.mjs"
+    }
+  ],
+  "validationChecks": [
+    {
+      "type": "file-exists",
+      "path": "index.html"
+    },
+    {
+      "type": "file-exists",
+      "path": "src/main.ts"
+    },
+    {
+      "type": "file-exists",
+      "path": "package.json"
+    },
+    {
+      "type": "command-success",
+      "command": [
+        "node",
+        "validate-threejs.mjs"
+      ],
+      "expect": "threejs game starter ok"
+    }
+  ],
+  "contextHints": {
+    "commands": [
+      "pnpm install",
+      "pnpm dev",
+      "pnpm build"
+    ],
+    "entrypoints": [
+      "src/main.ts",
+      "index.html",
+      "vite.config.ts",
+      "package.json"
+    ],
+    "preview": {
+      "path": "/"
+    },
+    "extensionPoints": [
+      "src/main.ts",
+      "src/scene.ts",
+      "src/loop.ts"
+    ]
+  },
+  "keywords": [
+    "three.js",
+    "threejs",
+    "webgl",
+    "3d scene",
+    "3d viewer",
+    "3d model",
+    "gltf",
+    "webgpu",
+    "shader",
+    "glsl"
+  ],
+  "scoring": {
+    "boost": {
+      "three.js": 4,
+      "threejs": 4,
+      "webgl": 2
+    }
+  },
+  "tieredKeywords": {
+    "tier1": [
+      "three.js",
+      "threejs",
+      "webgl 3d scene",
+      "3d model viewer",
+      "gltf viewer"
+    ],
+    "tier2": [
+      "3d scene",
+      "3d game",
+      "webgl",
+      "shader",
+      "glsl",
+      "raymarching",
+      "point cloud"
+    ],
+    "archetypes": [
+      "3d model viewer",
+      "3d product configurator",
+      "3d scene editor",
+      "gltf viewer",
+      "webgl game",
+      "3d portfolio",
+      "shader playground",
+      "particle system",
+      "physics sandbox",
+      "first person demo",
+      "orbital camera demo",
+      "3d data visualization"
+    ]
+  },
+  "buildHints": {
+    "whenToUse": "3D scene viewers, WebGL games, shader playgrounds, gltf/fbx asset explorers, or any product archetype where the primary surface is a live 3D canvas.",
+    "firstSteps": [
+      "Install: `pnpm install` then `pnpm dev` — Vite serves on 5173 with HMR, the cube should render immediately.",
+      "The full scene lives in `src/main.ts` — scene, camera, renderer, lights, mesh, and animation loop are all in one file so you can fork cleanly.",
+      "Add 3D models via `three/examples/jsm/loaders/GLTFLoader.js` — place assets in `public/` (Vite serves it at the root) and load with `new GLTFLoader().load('/model.glb', ...)`.",
+      "Build: `pnpm build` emits static assets to `dist/` — deploy anywhere; there is no server.",
+      "For shaders, import `.glsl` as `?raw` or inline as template strings; there is no GLSL plugin by default."
+    ],
+    "gotchas": [
+      "Canvas must be resized on window resize — call `renderer.setSize(w, h)` AND `camera.aspect = w/h; camera.updateProjectionMatrix()`. Setting only CSS width/height stretches the image and does not update the projection.",
+      "`renderer.setPixelRatio(window.devicePixelRatio)` is required for sharp output on retina, but cap it at 2 — uncapped it tanks framerate on 3x phones.",
+      "`requestAnimationFrame` must be cancelled on teardown (HMR, route change), otherwise multiple loops run in parallel and CPU climbs. Store the handle and call `cancelAnimationFrame`.",
+      "Dispose geometries, materials, and textures manually when swapping scenes — three.js does not garbage-collect GPU memory. Use `geometry.dispose()`, `material.dispose()`, `texture.dispose()`."
+    ],
+    "architectureNotes": [
+      "Single renderer, single scene, single render loop — do not create a new `WebGLRenderer` per frame or per component; it leaks the GL context.",
+      "Keep render-loop work pure: mutate transforms in the loop, construct meshes/materials outside it. Allocations inside the loop cause GC pauses visible as jank."
+    ],
+    "placeholders": [
+      {
+        "path": "src/main.ts",
+        "description": "Scene bootstrap — renderer, camera, lights, and a rotating cube plus resize + animation loop. Replace the cube with the product's actual geometry or `GLTFLoader` call; keep the resize + dispose plumbing."
+      }
+    ]
+  }
+}

package/registry/families/triton-server/files/README-triton.md

@@ -0,0 +1,65 @@
+# Triton — operator guide
+
+NVIDIA Triton Inference Server, serving the models laid out in
+`model_repository/`. This scaffold ships an `identity` echo model so you
+can smoke-test the server before you add real weights.
+
+## Layout
+
+```
+model_repository/
+  <model_name>/
+    config.pbtxt        # inputs, outputs, backend, batching
+    1/                  # version dir (MUST be an integer)
+      model.onnx        # or model.plan (TRT), model.py (Python), ...
+```
+
+## First-run
+
+```bash
+docker compose up -d
+curl http://localhost:{{port}}/v2/health/ready    # wait for HTTP 200
+pip install -r requirements.txt
+python -m src.client                              # echoes via identity model
+```
+
+## Adding a real model
+
+```bash
+# ONNX example
+mkdir -p model_repository/resnet50/1
+cp /path/to/resnet50.onnx model_repository/resnet50/1/model.onnx
+cat > model_repository/resnet50/config.pbtxt <<'EOF'
+name: "resnet50"
+backend: "onnxruntime"
+max_batch_size: 32
+input  [{ name: "input"  data_type: TYPE_FP32  dims: [3, 224, 224] }]
+output [{ name: "output" data_type: TYPE_FP32  dims: [1000]        }]
+dynamic_batching { preferred_batch_size: [8, 16, 32] }
+EOF
+# With --model-control-mode=poll, Triton auto-loads within 30s.
+curl http://localhost:{{port}}/v2/models/resnet50/ready
+```
+
+## Production notes
+
+- **Pin the image tag** — `nvcr.io/nvidia/tritonserver:24.10-py3` or whatever
+  matches the CUDA version your TRT engines were built against.
+- **TRT engines are GPU-specific**. `.plan` files compiled for A100 will not
+  run on H100 (and vice versa). Rebuild per deploy target.
+- **Metrics** live at `http://localhost:{{metricsPort}}/metrics` — Prometheus
+  format, per-model latency histograms, queue depth, GPU utilization.
+- **Model control**: in prod, use `--model-control-mode=explicit` and
+  POST to `/v2/repository/models/<name>/load`. `poll` is handy in dev
+  but racy under rapid deploys.
+- **Ensembles**: model_repository supports `platform: "ensemble"` with
+  `ensemble_scheduling` steps for preprocess → model → postprocess pipelines
+  that run entirely inside Triton.
+
+## Picking between Triton and the LLM servers
+
+- **Triton**: classical ML (CV/ASR/embeddings/rec), ONNX/TRT, mixed pipelines.
+- **vLLM / SGLang / TGI**: pure LLM serving, faster for that specific shape.
+
+Use Triton when the model isn't an LLM or when you're combining LLM + vision
+encoders + classical postprocess in a single-server pipeline.

package/registry/families/triton-server/files/client.py

@@ -0,0 +1,54 @@
+"""Triton client — hits the HTTP (KServe v2) endpoint on port {{port}}.
+
+Smoke-tests the `identity` model shipped in model_repository/. Swap
+`MODEL_NAME` and the input shape once you drop real weights in.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import numpy as np
+import tritonclient.http as httpclient
+
+
+TRITON_URL = os.environ.get("TRITON_URL", "localhost:{{port}}")
+MODEL_NAME = os.environ.get("MODEL_NAME", "identity")
+
+
+def ensure_ready(client: httpclient.InferenceServerClient) -> None:
+    if not client.is_server_live():
+        raise RuntimeError(f"Triton not live at {TRITON_URL}")
+    if not client.is_server_ready():
+        raise RuntimeError(f"Triton not ready at {TRITON_URL}")
+    if not client.is_model_ready(MODEL_NAME):
+        meta = client.get_model_metadata(MODEL_NAME)
+        raise RuntimeError(f"model {MODEL_NAME} not ready; metadata={meta}")
+
+
+def main() -> int:
+    client = httpclient.InferenceServerClient(url=TRITON_URL, verbose=False)
+
+    try:
+        ensure_ready(client)
+    except Exception as exc:
+        print(f"Triton not ready: {exc}", file=sys.stderr)
+        print("Did `docker compose up -d` finish? Check /v2/health/ready.", file=sys.stderr)
+        return 1
+
+    # The identity model echoes its input. 4 float32 elements, batch=1.
+    input_data = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32)
+    inputs = [httpclient.InferInput("INPUT0", input_data.shape, "FP32")]
+    inputs[0].set_data_from_numpy(input_data)
+    outputs = [httpclient.InferRequestedOutput("OUTPUT0")]
+
+    response = client.infer(model_name=MODEL_NAME, inputs=inputs, outputs=outputs)
+    result = response.as_numpy("OUTPUT0")
+    print(f"input : {input_data.flatten().tolist()}")
+    print(f"output: {result.flatten().tolist()}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/registry/families/triton-server/files/docker-compose.yml

@@ -0,0 +1,41 @@
+# NVIDIA Triton Inference Server.
+# Pin the tritonserver tag (e.g. 24.10-py3) — `latest` can break TRT engines
+# that were built for a specific CUDA/TRT minor.
+services:
+  triton:
+    image: nvcr.io/nvidia/tritonserver:latest
+    container_name: {{serviceName}}
+    restart: unless-stopped
+    ports:
+      - "{{port}}:8000"        # HTTP
+      - "{{grpcPort}}:8001"    # gRPC
+      - "{{metricsPort}}:8002" # Prometheus metrics
+    shm_size: 1gb
+    environment:
+      # Log verbosity. 0 = errors, 1 = info, 2 = debug.
+      TRITON_LOG_VERBOSE: "0"
+    command:
+      - tritonserver
+      - --model-repository=/models
+      - --strict-model-config=false
+      - --allow-metrics=true
+      # --model-control-mode=poll re-scans /models every N seconds, so you
+      # can hot-add models without restarting. Use `explicit` in prod and
+      # POST /v2/repository/models/<name>/load manually.
+      - --model-control-mode=poll
+      - --repository-poll-secs=30
+    volumes:
+      - ./model_repository:/models
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/v2/health/ready"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s

package/registry/families/triton-server/files/model_repository/identity/1/.gitkeep

@@ -0,0 +1,9 @@
+# Version directory for the `identity` model. MUST be an integer (`1`, `2`, ...).
+# Triton silently skips non-integer dirs like `v1` or `latest`.
+#
+# The `identity` backend needs no weight file. For a real model, drop the
+# weights here, e.g.:
+#
+#   model_repository/resnet50/1/model.onnx
+#   model_repository/bert-large/1/model.plan
+#   model_repository/preprocess/1/model.py

package/registry/families/triton-server/files/model_repository/identity/config.pbtxt

@@ -0,0 +1,43 @@
+# Identity/echo model — exists so `docker compose up` + `python -m src.client`
+# works out-of-the-box without shipping real weights. Replace with your real
+# model by:
+#   1) dropping weights into ../<version>/   (e.g. model.onnx, model.plan)
+#   2) changing `backend:` to match (onnxruntime | tensorrt | python | pytorch)
+#   3) updating input/output dims to your model's actual shape
+#
+# Triton ships a built-in `identity` backend that echoes inputs verbatim —
+# no weight file is required for this version.
+
+name: "identity"
+backend: "identity"
+max_batch_size: 8
+
+input [
+  {
+    name: "INPUT0"
+    data_type: TYPE_FP32
+    dims: [ 4 ]
+  }
+]
+
+output [
+  {
+    name: "OUTPUT0"
+    data_type: TYPE_FP32
+    dims: [ 4 ]
+  }
+]
+
+# Dynamic batching — Triton coalesces concurrent single-sample requests.
+# Tune max_queue_delay_microseconds per latency SLO.
+dynamic_batching {
+  preferred_batch_size: [ 4, 8 ]
+  max_queue_delay_microseconds: 1000
+}
+
+instance_group [
+  {
+    count: 1
+    kind: KIND_GPU
+  }
+]

package/registry/families/triton-server/files/requirements.txt

@@ -0,0 +1,2 @@
+tritonclient[http]>=2.48.0
+numpy>=1.26.0,<2.0.0