@oh-my-pi/pi-catalog 17.3.8 → 17.4.0

package/dist/types/discovery/index.d.ts

@@ -3,3 +3,4 @@ export * from "./codex.js";
 export * from "./gemini.js";
 export * from "./gitlab-duo-workflow.js";
 export * from "./openai-compatible.js";
+export * from "./protobuf.js";

package/dist/types/discovery/protobuf.d.ts

@@ -0,0 +1,102 @@
+/**
+ * High-performance, zero-builder protobuf wire codecs for @oh-my-pi/pi-catalog.
+ *
+ * Schemas are declared as static IR descriptors with near-zero module load overhead
+ * and lazy compilation on first encode/decode/create invocation.
+ */
+/** JSON values carried by `google.protobuf.Value` fields. */
+export type JsonValue = null | boolean | number | string | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+/** An unrecognised wire field retained for forward-compatible round-trips. */
+export interface ProtoUnknownField {
+    no: number;
+    wireType: number;
+    data: Uint8Array;
+}
+/** Shared internal metadata present on every decoded protocol message. */
+export interface ProtoMessage {
+    $typeName?: string;
+    $unknown?: ProtoUnknownField[];
+}
+/** A bidirectional codec for one protobuf message type. */
+export interface MessageCodec<T extends ProtoMessage = ProtoMessage> {
+    (value: T): Uint8Array;
+    (value: Uint8Array): T;
+    /** Creates a message with protobuf defaults for omitted fields. */
+    create(value?: Partial<T>): T;
+    /** Encodes one message into protobuf wire bytes. */
+    encode(value: T): Uint8Array;
+    /** Decodes one protobuf message from wire bytes. */
+    decode(value: Uint8Array): T;
+    /** Converts a message to its protobuf JSON representation. */
+    toJson(value: T): JsonValue;
+}
+/** Infers a message shape from a codec result. */
+export type InferMessage<TCodec> = TCodec extends MessageCodec<infer TMessage> ? TMessage : never;
+/** Erases a referenced message's concrete shape for static field descriptors. */
+export interface MessageReference {
+    encode(value: unknown): Uint8Array;
+    decode(value: Uint8Array): ProtoMessage;
+    toJson(value: unknown): JsonValue;
+}
+export type ScalarKind = "bool" | "bytes" | "double" | "enum" | "float" | "int32" | "int64" | "string" | "uint32" | "uint64";
+export type WireType = 0 | 1 | 2 | 5;
+export interface ScalarFieldDesc {
+    readonly no: number;
+    readonly name: string;
+    readonly kind: ScalarKind;
+    readonly optional?: boolean;
+    readonly repeat?: boolean;
+}
+export interface MessageFieldDesc {
+    readonly no: number;
+    readonly name: string;
+    readonly kind: "message";
+    readonly T: () => MessageReference;
+    readonly repeat?: boolean;
+}
+export interface EnumFieldDesc {
+    readonly no: number;
+    readonly name: string;
+    readonly kind: "enum";
+    readonly optional?: boolean;
+    readonly repeat?: boolean;
+}
+export interface MapFieldDesc {
+    readonly no: number;
+    readonly name: string;
+    readonly kind: "map";
+    readonly K: "string";
+    readonly V: ScalarKind | (() => MessageReference);
+}
+export type VariantDesc = {
+    readonly no: number;
+    readonly name: string;
+    readonly kind: ScalarKind;
+} | {
+    readonly no: number;
+    readonly name: string;
+    readonly kind: "message";
+    readonly T: () => MessageReference;
+};
+export interface OneofFieldDesc {
+    readonly kind: "oneof";
+    readonly name: string;
+    readonly variants: readonly VariantDesc[];
+}
+export type FieldDesc = ScalarFieldDesc | MessageFieldDesc | EnumFieldDesc | MapFieldDesc | OneofFieldDesc;
+/** Creates a high-performance, lazy protobuf message codec from an IR field descriptor list. */
+export declare function pb<T extends ProtoMessage = ProtoMessage>(typeName: string, fields?: readonly FieldDesc[]): MessageCodec<T>;
+/** Creates a message using its codec's protobuf defaults. */
+export declare function create<TMessage extends ProtoMessage>(codec: MessageCodec<TMessage>, value?: Partial<TMessage>): TMessage;
+/** Encodes a message using its codec. */
+export declare function toBinary<TMessage extends ProtoMessage>(codec: MessageCodec<TMessage>, value: TMessage): Uint8Array;
+/** Decodes wire bytes using a message codec. */
+export declare function fromBinary<TMessage extends ProtoMessage>(codec: MessageCodec<TMessage>, value: Uint8Array): TMessage;
+/** Converts a message to protobuf JSON using its codec. */
+export declare function toJson<TMessage extends ProtoMessage>(codec: MessageCodec<TMessage>, value: TMessage): JsonValue;
+/** Encodes a JSON value as `google.protobuf.Value`. */
+export declare function encodeJsonValue(value: JsonValue): Uint8Array;
+/** Decodes `google.protobuf.Value` wire bytes into a JSON value. */
+export declare function decodeJsonValue(value: Uint8Array): JsonValue;

package/dist/types/identity/family.d.ts

@@ -56,6 +56,8 @@ export declare const isDeepseekModelIdOrName: (modelId: string) => boolean;
 export declare const isDeepseekV4FlashModelId: (modelId: string) => boolean;
 /** Xiaomi MiMo family by id or display name. */
 export declare const isMimoModelIdOrName: (modelId: string) => boolean;
+/** StepFun Step 3.7 Flash SKU in any namespace form (`kilo/stepfun/step-3.7-flash:free`). */
+export declare const isStep37FlashModelId: (modelId: string) => boolean;
 /** Gemini family ids in any namespace form (`gemini-*`, `google/gemini-*`, `openrouter/google/gemini-…`). */
 export declare const isGeminiModelId: (modelId: string) => boolean;
 /** Grok family ids across namespace and delimiter forms (`grok-*`, `cursor-grok-*`, `xai/grok-*`). */
@@ -199,6 +201,14 @@ export declare const hasOpus47ApiRestrictions: (modelId: string) => boolean;
  * @see https://platform.claude.com/docs/en/build-with-claude/mid-conversation-system-messages
  */
 export declare const supportsMidConversationSystemMessages: (modelId: string) => boolean;
+/**
+ * Models that reliably follow the hashline line-anchored edit dialect
+ * (`[path#TAG]` headers plus 1-indexed anchors). Kimi, MiMo, DeepSeek V4
+ * Flash, and Step 3.7 Flash miscount anchors or drop the tag header often
+ * enough that hosts fall back to a literal search-replace edit format for
+ * them.
+ */
+export declare const supportsHashlineEdits: (modelId: string) => boolean;
 export declare const isAnthropicFableOrMythosModel: (modelId: string) => boolean;
 /** Thinking-variant token location inside a model id. */
 export interface ThinkingVariantToken {

package/dist/types/index.d.ts

@@ -6,6 +6,7 @@ export * from "./identity/index.js";
 export * from "./model-cache.js";
 export * from "./model-manager.js";
 export * from "./model-thinking.js";
+export * from "./model-tokenizer.js";
 export * from "./models.js";
 export * from "./provider-models/index.js";
 export * from "./types.js";

package/dist/types/model-tokenizer.d.ts

@@ -0,0 +1,8 @@
+import type { ModelTokenizer } from "./types.js";
+/**
+ * Resolve the exact locally embedded tokenizer for a canonical model id.
+ *
+ * This is catalog policy, not a runtime caller heuristic: [`buildModel`](./build.ts)
+ * materializes the result as `Model.tokenizer`; consumers read that property.
+ */
+export declare function resolveModelTokenizer(modelId: string): ModelTokenizer | undefined;

package/dist/types/types.d.ts

@@ -716,6 +716,15 @@ export interface LongContextTokenCost extends TokenCost {
 export interface ModelCost extends TokenCost {
     longContext?: LongContextTokenCost;
 }
+/**
+ * Exact local content tokenizer family for a model.
+ *
+ * Absent means no first-party local tokenizer is known and consumers retain
+ * their estimate/default-tokenizer policy. The values name tokenizer
+ * generations rather than providers: DeepSeek V3 through V4 share
+ * `"deepseek-v3"`; Kimi K2 through K3 share `"kimi-k2"`.
+ */
+export type ModelTokenizer = "claude-v3" | "claude-v47" | "claude-v5" | "claude-v5-sonnet" | "qwen3" | "deepseek-v3" | "kimi-k2" | "glm5";
 export interface Model<TApi extends Api = Api> {
     id: string;
     /**
@@ -739,6 +748,12 @@ export interface Model<TApi extends Api = Api> {
     provider: Provider;
     baseUrl: string;
     reasoning: boolean;
+    /**
+     * Exact local tokenizer family resolved from the model identity or supplied
+     * explicitly by a catalog/discovery source. Absent leaves local counting to
+     * the consumer's fallback policy.
+     */
+    tokenizer?: ModelTokenizer;
     input: ("text" | "image")[];
     /**
      * Decoder family used for image inputs when it has narrower format support

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-catalog",
-	"version": "17.3.8",
+	"version": "17.4.0",
 	"description": "Model catalog for omp: bundled model database, provider discovery descriptors, model identity, classification, and equivalence",
 	"homepage": "https://omp.sh",
 	"author": "Can Boluk",
@@ -31,15 +31,14 @@
 		"fix": "biome check --write --unsafe .",
 		"fmt": "biome format --write .",
 		"gen:models": "bun scripts/generate-models.ts",
-		"gen:cursor-proto": "protoc --plugin=protoc-gen-es=../../node_modules/.bin/protoc-gen-es --es_out=src/discovery/cursor-gen --es_opt=target=ts -I ../ai/src/providers/cursor/proto ../ai/src/providers/cursor/proto/agent.proto"
+		"gen:proto": "bun scripts/generate-protocols.ts"
 	},
 	"dependencies": {
-		"@bufbuild/protobuf": "^2.12.1",
-		"@oh-my-pi/omptype": "17.3.8",
-		"@oh-my-pi/pi-utils": "17.3.8"
+		"@oh-my-pi/omptype": "17.4.0",
+		"@oh-my-pi/pi-utils": "17.4.0"
 	},
 	"devDependencies": {
-		"@oh-my-pi/pi-ai": "17.3.8",
+		"@oh-my-pi/pi-ai": "17.4.0",
 		"@types/bun": "^1.3.14"
 	},
 	"engines": {

package/src/build.ts

@@ -16,6 +16,7 @@ import { buildDevinCompat } from "./compat/devin";
 import { buildOpenAICompat, buildOpenAIResponsesCompat, buildOpenRouterCompat } from "./compat/openai";
 import { bareModelId, parseOpenAIModel, semverGte } from "./identity/classify";
 import { resolveModelThinking } from "./model-thinking";
+import { resolveModelTokenizer } from "./model-tokenizer";
 import type { Api, CompatOf, Model, ModelSpec } from "./types";
 import { cleanModelName } from "./utils";
 
@@ -58,12 +59,18 @@ function supportsOpenAIGAComputerUse(spec: ModelSpec<Api>, explicitSupport: bool
 	return parsed !== null && semverGte(parsed.version, "5.4");
 }
 
+/**
+ * Build one model from an authored spec. Bundled models.json rows are fully
+ * materialized by the generator and consumed directly (see `models.ts`), so
+ * this only runs for discovered/custom/override specs.
+ */
 export function buildModel<TApi extends Api>(spec: ModelSpec<TApi>): Model<TApi> {
 	const compat = buildCompat(spec) as CompatOf<TApi>;
 	const supportsComputerUseConfig = explicitComputerUseConfig(spec);
 	return {
 		...spec,
 		name: cleanModelName(spec.name),
+		tokenizer: spec.tokenizer ?? resolveModelTokenizer(spec.requestModelId ?? spec.id),
 		thinking: resolveModelThinking(spec, compat),
 		supportsComputerUse: supportsOpenAIGAComputerUse(spec, supportsComputerUseConfig),
 		supportsComputerUseConfig,