@kt3k/tku 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright 2026 Yoshiya Hinosawa ( @kt3k )
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/README.md

@@ -0,0 +1,81 @@
+# @kt3k/tku — Git Repository Token Counter
+
+## Overview
+
+`@kt3k/tku` is a CLI tool that counts the total number of tokens in a git
+repository. It uses [tiktoken](https://www.npmjs.com/package/tiktoken) to
+tokenize file contents and reports the token count per file and in total.
+
+## Motivation
+
+When working with LLMs, knowing the token size of a codebase helps estimate
+context window usage, cost, and whether a repository (or subset) fits within
+model limits. `tku` provides a quick, accurate measurement directly from the
+command line.
+
+## Usage
+
+```sh
+npx @kt3k/tku [options] [path]
+```
+
+- `path` — Path to a git repository (defaults to current directory).
+
+### Options
+
+| Flag                      | Description                                          | Default              |
+| ------------------------- | ---------------------------------------------------- | -------------------- |
+| `--encoding <encoding>`   | Tiktoken encoding (e.g. `o200k_base`, `cl100k_base`) | `o200k_base`         |
+| `-e, --exclude <glob...>` | Additional glob patterns to exclude                  | none                 |
+| `--no-gitignore`          | Do not respect `.gitignore` rules                    | respect `.gitignore` |
+| `--json`                  | Output results as JSON                               | false                |
+| `--top <n>`               | Show only the top N files by token count             | show all             |
+| `--sort <field>`          | Sort by `tokens` or `path`                           | `tokens`             |
+
+### Examples
+
+```sh
+# Count tokens in the current repo
+npx @kt3k/tku
+
+# Count tokens using cl100k_base encoding, show top 20 files
+npx @kt3k/tku --encoding cl100k_base --top 20
+
+# JSON output for scripting
+npx @kt3k/tku --json
+
+# Exclude test files
+npx @kt3k/tku --exclude "**/*.test.*" --exclude "**/fixtures/**"
+```
+
+## Output
+
+### Default (human-readable)
+
+```
+tokens  path
+ 1.2 K  src/index.ts
+   892  src/utils.ts
+   345  README.md
+──────
+ 2.4 K  total (3 files)
+```
+
+### JSON (`--json`)
+
+```json
+{
+  "encoding": "o200k_base",
+  "files": [
+    { "path": "src/index.ts", "tokens": 1204 },
+    { "path": "src/utils.ts", "tokens": 892 },
+    { "path": "README.md", "tokens": 345 }
+  ],
+  "totalTokens": 2441,
+  "totalFiles": 3
+}
+```
+
+## License
+
+MIT

package/dist/main.js

@@ -0,0 +1,218 @@
+// src/main.ts
+import process from "node:process";
+import { parseArgs } from "node:util";
+
+// src/files.ts
+import { resolve } from "node:path";
+import { open } from "node:fs/promises";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+var execFileAsync = promisify(execFile);
+async function listFiles(repoPath, options = {}) {
+  const cwd = resolve(repoPath);
+  const args = [
+    "ls-files"
+  ];
+  if (options.noGitignore) {
+    args.push("--cached", "--others", "--exclude-standard");
+  }
+  let stdout;
+  try {
+    const result = await execFileAsync("git", args, {
+      cwd
+    });
+    stdout = result.stdout;
+  } catch (e) {
+    const msg = e.stderr?.trim() ?? String(e);
+    throw new Error(`Not a git repository or git error: ${msg}`);
+  }
+  let files = stdout.trim().split("\n").filter((f) => f.length > 0);
+  if (options.exclude && options.exclude.length > 0) {
+    const patterns = options.exclude.map((g) => new URLPattern({
+      pathname: g
+    }));
+    files = files.filter((f) => !patterns.some((p) => p.test({
+      pathname: f
+    })));
+  }
+  return files;
+}
+async function isBinary(filePath) {
+  const handle = await open(filePath, "r");
+  try {
+    const buf = new Uint8Array(8192);
+    const { bytesRead } = await handle.read(buf);
+    if (bytesRead === null || bytesRead === 0) return false;
+    for (let i = 0; i < bytesRead; i++) {
+      if (buf[i] === 0) return true;
+    }
+    return false;
+  } finally {
+    await handle.close();
+  }
+}
+async function listTextFiles(repoPath, options = {}) {
+  const cwd = resolve(repoPath);
+  const files = await listFiles(repoPath, options);
+  const results = [];
+  for (const file of files) {
+    const fullPath = resolve(cwd, file);
+    try {
+      if (!await isBinary(fullPath)) {
+        results.push(file);
+      }
+    } catch {
+    }
+  }
+  return results;
+}
+
+// src/tokenize.ts
+import { readFile } from "node:fs/promises";
+import { resolve as resolve2 } from "node:path";
+import { get_encoding } from "tiktoken";
+async function tokenizeFiles(repoPath, files, encoding) {
+  const enc = get_encoding(encoding);
+  try {
+    const results = [];
+    let totalTokens = 0;
+    for (const file of files) {
+      const fullPath = resolve2(repoPath, file);
+      const content = await readFile(fullPath, "utf-8");
+      const tokens = enc.encode_ordinary(content).length;
+      results.push({
+        path: file,
+        tokens
+      });
+      totalTokens += tokens;
+    }
+    return {
+      encoding,
+      files: results,
+      totalTokens,
+      totalFiles: results.length
+    };
+  } finally {
+    enc.free();
+  }
+}
+
+// src/format.ts
+function formatTokenCount(n) {
+  if (n >= 1e6) {
+    return `${(n / 1e6).toFixed(1)} M`;
+  }
+  if (n >= 1e3) {
+    return `${(n / 1e3).toFixed(1)} K`;
+  }
+  return String(n);
+}
+function formatTable(result) {
+  const lines = [];
+  const formatted = result.files.map((f) => ({
+    path: f.path,
+    display: formatTokenCount(f.tokens)
+  }));
+  const maxWidth = Math.max("tokens".length, ...formatted.map((f) => f.display.length));
+  lines.push(`${"tokens".padStart(maxWidth)}  path`);
+  for (const f of formatted) {
+    lines.push(`${f.display.padStart(maxWidth)}  ${f.path}`);
+  }
+  lines.push(`${"\u2500".repeat(maxWidth)}\u2500\u2500`);
+  const totalDisplay = formatTokenCount(result.totalTokens);
+  lines.push(`${totalDisplay.padStart(maxWidth)}  total (${result.totalFiles} files)`);
+  return lines.join("\n");
+}
+function formatResult(result, options = {}) {
+  const { json = false, top, sort = "tokens" } = options;
+  const sorted = [
+    ...result.files
+  ];
+  if (sort === "path") {
+    sorted.sort((a, b) => a.path.localeCompare(b.path));
+  } else {
+    sorted.sort((a, b) => b.tokens - a.tokens);
+  }
+  const filtered = top !== void 0 ? sorted.slice(0, top) : sorted;
+  const adjusted = {
+    ...result,
+    files: filtered
+  };
+  return json ? JSON.stringify(adjusted, null, 2) : formatTable(adjusted);
+}
+
+// src/main.ts
+function printUsage() {
+  console.log(`Usage: tku [options] [path]
+
+Options:
+      --encoding <encoding>   Tiktoken encoding (default: o200k_base)
+  -e, --exclude <glob...>     Glob patterns to exclude (repeatable)
+      --no-gitignore          Do not respect .gitignore rules
+      --json                  Output results as JSON
+      --top <n>               Show only the top N files by token count
+      --sort <field>          Sort by "tokens" or "path" (default: tokens)
+  -h, --help                  Show this help message`);
+}
+async function main() {
+  const { values, positionals } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      encoding: {
+        type: "string",
+        default: "o200k_base"
+      },
+      exclude: {
+        type: "string",
+        short: "e",
+        multiple: true
+      },
+      gitignore: {
+        type: "boolean",
+        default: true
+      },
+      json: {
+        type: "boolean",
+        default: false
+      },
+      top: {
+        type: "string"
+      },
+      sort: {
+        type: "string",
+        default: "tokens"
+      },
+      help: {
+        type: "boolean",
+        short: "h",
+        default: false
+      }
+    },
+    allowPositionals: true
+  });
+  if (values.help) {
+    printUsage();
+    process.exit(0);
+  }
+  const repoPath = positionals[0] ?? ".";
+  const encoding = values.encoding;
+  const sort = values.sort;
+  const top = values.top !== void 0 ? Number(values.top) : void 0;
+  try {
+    const files = await listTextFiles(repoPath, {
+      exclude: values.exclude,
+      noGitignore: !values.gitignore
+    });
+    const result = await tokenizeFiles(repoPath, files, encoding);
+    const output = formatResult(result, {
+      json: values.json,
+      top,
+      sort
+    });
+    console.log(output);
+  } catch (e) {
+    console.error(e instanceof Error ? e.message : String(e));
+    process.exit(1);
+  }
+}
+main();

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@kt3k/tku",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/main.js",
+  "bin": {
+    "tku": "dist/main.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "deno bundle --external tiktoken -o dist/main.js src/main.ts",
+    "prepublishOnly": "npm run build",
+    "test": "vitest run"
+  },
+  "keywords": [],
+  "author": "Yoshiya Hinosawa",
+  "license": "MIT",
+  "type": "module",
+  "dependencies": {
+    "tiktoken": "^1"
+  },
+  "devDependencies": {
+    "@types/node": "^25",
+    "string-dedent": "^3",
+    "vitest": "^3"
+  }
+}