language-models 0.1.0

package/generate/build-models.ts

@@ -0,0 +1,150 @@
+import camelCase from 'camelcase'
+import { flatten, unflatten } from 'flat'
+
+const overwrites = {}
+
+const sortingValues = ['top-weekly', 'newest', 'throughput-high-to-low', 'latency-low-to-high', 'pricing-low-to-high', 'pricing-high-to-low']
+
+function camelCaseDeep<T>(input: T): T {
+  if (Array.isArray(input)) {
+    return input.map(camelCaseDeep) as any
+  }
+
+  if (input !== null && typeof input === 'object') {
+    return Object.entries(input).reduce(
+      (acc, [key, value]) => {
+        acc[camelCase(key)] = camelCaseDeep(value)
+        return acc
+      },
+      {} as Record<string, any>,
+    ) as any
+  }
+
+  return input
+}
+
+async function fetchProviders(slug: string) {
+  const url = `https://openrouter.ai/api/frontend/stats/endpoint?permaslug=${slug}`
+
+  const response = (await fetch(url).then((res) => res.json())) as { data: any }
+
+  return camelCaseDeep(response.data || [])
+}
+
+async function main() {
+  try {
+    const URL = 'https://openrouter.ai/api/frontend/models/find?order=top-weekly'
+
+    console.log('Fetching models data from OpenRouter...')
+
+    const data = (
+      await Promise.all(
+        sortingValues.map(async (sortingValue) => {
+          const URL = `https://openrouter.ai/api/frontend/models/find?order=${sortingValue}`
+          console.log(`Fetching ${URL}...`)
+          const response = (await fetch(URL).then((res) => res.json())) as { data: any }
+          return [sortingValue, camelCaseDeep(response.data)]
+        }),
+      )
+    ).reduce(
+      (acc, [sortingValue, models]) => {
+        acc[sortingValue] = models
+        return acc
+      },
+      {} as Record<string, any>,
+    )
+
+    const models = data['top-weekly']
+
+    const modelsData = models.models.map((model) => {
+      if (model.slug in overwrites) {
+        console.log(`Overwriting model ${model.slug} with custom data`, overwrites[model.slug])
+        const tempModel: Record<string, unknown> = flatten(model)
+        const tempOverwrites: Record<string, unknown> = flatten(overwrites[model.slug])
+        const mergedModel = { ...tempModel, ...tempOverwrites }
+        return unflatten(mergedModel)
+      }
+
+      // Find the model in the other sorting values
+      // Then add a sorting object that has the index of the model in the other sorting values
+
+      const modelIndexes = sortingValues
+        .map((sortingValue) => {
+          const models = data[sortingValue]
+          const index = models.models.findIndex((m) => m.slug === model.slug)
+          return {
+            [camelCase(sortingValue)]: index,
+          }
+        })
+        .reduce((acc, curr) => {
+          return { ...acc, ...curr }
+        }, {})
+
+      const mergedModel = { ...model, sorting: modelIndexes }
+
+      return mergedModel
+    })
+
+    const finalModels = []
+
+    const providerAliases = {
+      'Google AI Studio': 'google',
+      'Google Vertex': 'vertex',
+    }
+
+    let completed = 0
+
+    for (const model of modelsData) {
+      console.log(`[PROVIDERS] Fetching provider metadata for ${model.permaslug}...`)
+
+      const providers = await fetchProviders(model.permaslug)
+      model.providers = providers.map((provider) => {
+        const providerName = providerAliases[provider.providerDisplayName] || provider.providerDisplayName
+
+        const priceToDollars = (price: string) => {
+          // To get the dollar price, we need to multiply the price by a million.
+          const priceNumber = parseFloat(price)
+          return Number(
+            (priceNumber * 1000000)
+              .toFixed(2)
+              // Remove trailing zeros
+              .replace(/\.?0+$/, ''),
+          )
+        }
+
+        return {
+          name: provider.providerDisplayName,
+          slug: camelCase(providerName),
+          quantization: provider.quantization,
+          context: provider.contextLength,
+          maxCompletionTokens: provider.maxCompletionTokens,
+          providerModelId: provider.providerModelId,
+          pricing: provider.pricing,
+          // Disable claude's reasoning parameter as it's only supported via the :thinking tag.
+          supportedParameters: model.slug === 'anthropic/claude-3.7-sonnet' ? ['max_tokens', 'temperature', 'stop', 'tools', 'tool_choice'] : provider.supportedParameters,
+          inputCost: priceToDollars(provider.pricing.prompt),
+          outputCost: priceToDollars(provider.pricing.completion),
+          throughput: provider.stats?.[0]?.p50Throughput,
+          latency: provider.stats?.[0]?.p50Latency,
+        }
+      })
+
+      completed++
+      console.log(`[PROVIDERS] ${completed}/${modelsData.length} models completed`)
+    }
+
+    // Write to models.json in src directory
+    const { resolve } = await import('node:path')
+    const { writeFileSync } = await import('node:fs')
+
+    const outputPath = resolve('./src/models.js')
+    writeFileSync(outputPath, `export default ${JSON.stringify({ models: modelsData }, null, 2)}`)
+
+    console.log(`Models data written to ${outputPath}`)
+  } catch (error) {
+    console.error('Error fetching or writing models data:', error)
+    process.exit(1)
+  }
+}
+
+main()

package/generate/overwrites.ts

@@ -0,0 +1,12 @@
+/*
+ * This file contains overwrites for the AI models.
+ * OpenRouter's data is not always 100% accurate, so we can overwrite certain
+ * models with our own data.
+ */
+
+export const overwrites = {
+  'anthropic/claude-3.7-sonnet': {
+    // Explicitly remove reasoning from this model as it's not actually supported.
+    'endpoint.supportedParameters': ['max_tokens', 'temperature', 'stop', 'tools', 'tool_choice'],
+  },
+}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "language-models",
+  "version": "0.1.0",
+  "description": "Language model abstractions",
+  "main": "dist/index.js",
+  "scripts": {
+    "dev": "dotenvx run -- npx tsx watch src/demo.ts",
+    "build": "rimraf dist && tsc && rm dist/models.d.ts && cp src/models.d.ts dist/",
+    "generate:models": "npx tsx generate/build-models.ts",
+    "clean": "rimraf dist",
+    "lint": "tsc --noEmit",
+    "prepublishOnly": "pnpm clean && pnpm build",
+    "test": "vitest",
+    "typecheck": "tsc --noEmit"
+  },
+  "type": "module",
+  "packageManager": "pnpm@10.6.5",
+  "devDependencies": {
+    "@types/node": "^22.13.10",
+    "rimraf": "^6.0.1",
+    "typescript": "^5.8.2",
+    "vitest": "^3.0.9",
+    "tsconfig": "0.0.0",
+    "eslint-config": "0.1.0"
+  },
+  "dependencies": {
+    "@types/node": "^22.13.10",
+    "camelcase": "^8.0.0",
+    "flat": "^6.0.1",
+    "vitest": "^3.0.9"
+  }
+}

package/publish.js

@@ -0,0 +1,32 @@
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+try {
+  console.log('Attempting direct NPM publish...');
+  
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = path.dirname(__filename);
+  const pkgPath = path.join(process.cwd(), 'package.json');
+  const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+  
+  if (!pkg.version.startsWith('0.')) {
+    console.log(`Fixing package.json version: ${pkg.version} -> 0.1.0`);
+    pkg.version = '0.1.0';
+    fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n', 'utf8');
+  }
+  
+  execSync('npm publish --access public --tag next', { 
+    stdio: 'inherit',
+    env: {
+      ...process.env,
+      NPM_CONFIG_REGISTRY: 'https://registry.npmjs.org/',
+    }
+  });
+  
+  console.log('Direct NPM publish successful');
+} catch (error) {
+  console.error('Error during direct NPM publish:', error.message);
+}

package/roadmap.md

@@ -0,0 +1,54 @@
+# `ai-models` Roadmap
+
+This roadmap outlines the planned features and considerations for the `ai-models` package, which provides utilities for working with AI SDKs, functions, workflows, observability, and evaluations.
+
+## Model Identification & Syntax
+
+- [ ] Support for provider/creator/model naming convention
+- [ ] Determine whether to use the @ sign in naming syntax
+- [ ] Support for creator/model syntax without provider specification
+- [ ] Alignment with OpenRouter's syntax where possible
+- [ ] Design naming convention: `@{provider}/{creator}/{model}:{config,capabilities,tools,priorities}`
+- [ ] Alternative syntax support: `{creator}/{model}:{config,capabilities,tools,priorities}`
+
+## Routing & Model Selection
+
+- [ ] Implement our own version of openrouter/auto and openrouter/auto:online
+- [ ] Tie routing logic to message content and/or priorities (performance, latency, throughput, cost)
+- [ ] Support dynamic model fallback pattern
+- [ ] Enable runtime tweaking of provider/model/settings/tools via query parameters
+
+## Capabilities & Features
+
+- [ ] Handle reasoning capabilities (follow OpenRouter's :reasoning flag)
+- [ ] Support for code execution tools
+- [ ] Design composite type tools for transforming outputs between models
+- [ ] Implement online search capability routing (to models like gemini, perplexity, 4o-search)
+- [ ] Support for general-purpose tools (similar to agentic.so)
+- [ ] Develop secure code execution tool for JavaScript (as alternative to Python-based tools)
+
+## Structured Output & Response Formats
+
+- [ ] Support various methods for structured outputs:
+  - [ ] Native structured_output support
+  - [ ] Tool use with schema enforcement
+  - [ ] Response_format with JSON guarantees
+  - [ ] System prompt fallback method
+- [ ] Handle compatibility between different output methods
+
+## Use Cases Implementation
+
+- [ ] Evaluation framework with dynamic model/settings/tools configuration
+- [ ] Experimentation support (LLM-as-Judge comparisons between models)
+- [ ] "Best model" specification without cost/price requirements
+- [ ] Specific capability routing (e.g., reasoning-only models)
+- [ ] Caching opt-in/out functionality
+- [ ] Logging controls for sensitive PII situations
+- [ ] Seed parameter support
+
+## Phased Implementation
+
+- [ ] Phase 1: Core model identification and routing
+- [ ] Phase 2: Structured output and basic capabilities
+- [ ] Phase 3: Advanced features and tools integration
+- [ ] Phase 4: Comprehensive evaluation and experimentation framework

package/src/aliases.ts

@@ -0,0 +1,5 @@
+export const aliases: Record<string, string> = {
+  gemini: 'google/gemini-2.0-flash-001',
+  'claude-3.7-sonnet': 'anthropic/claude-3.7-sonnet',
+  r1: 'deepseek/deepseek-r1',
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export * from './parser'
+export * from './aliases'
+export * from './providers'
+import allModels from './models'
+import type { Model } from './types'
+export * from './types'
+
+export const models = allModels.models as unknown as Model[]
+
+export const modelPattern = /^(?:[\w-]+\/)?[\w\.-]+(?:@[\w-]+(?::[\w-]+)?)?(?<!@)\([^()]*(?:\([^()]*\)[^()]*)*\)$/

package/src/models.d.ts

@@ -0,0 +1,170 @@
+// --- Reusable Base Interfaces ---
+
+interface Icon {
+  url: string
+  invertRequired?: boolean
+}
+
+interface DataPolicy {
+  termsOfServiceUrl?: string // Optional based on examples
+  privacyPolicyUrl?: string // Optional based on examples
+  training: boolean
+  loggingRetentionDays?: number // Optional based on examples
+  requiresUserIds?: boolean // Optional based on examples
+}
+
+// Base pricing structure, allowing for optional cache fields
+interface Pricing {
+  prompt: string
+  completion: string
+  image: string
+  request: string
+  webSearch: string
+  internalReasoning: string
+  discount: number
+  inputCacheRead?: string // Optional based on examples
+  inputCacheWrite?: string // Optional based on examples
+}
+
+interface ReasoningConfig {
+  startToken: string
+  endToken: string
+}
+
+interface Sorting {
+  topWeekly: number
+  newest: number
+  throughputHighToLow: number
+  latencyLowToHigh: number
+  pricingLowToHigh: number
+  pricingHighToLow: number
+}
+
+interface ProviderInfo {
+  name: string
+  displayName: string
+  baseUrl: string
+  dataPolicy: DataPolicy
+  headquarters?: string // Optional based on examples
+  hasChatCompletions: boolean
+  hasCompletions: boolean
+  isAbortable: boolean
+  moderationRequired: boolean
+  group: string
+  editors: never[] // Assuming these are always empty based on examples
+  owners: never[] // Assuming these are always empty based on examples
+  isMultipartSupported: boolean
+  statusPageUrl: string | null
+  byokEnabled: boolean
+  isPrimaryProvider: boolean
+  icon: Icon
+}
+
+// Represents an entry in the 'providers' array within a Model
+interface ModelProvider {
+  name: string
+  slug: string
+  quantization: string | null
+  context: number
+  maxCompletionTokens: number | null // Can be null in examples
+  pricing: Pricing // Use the common Pricing type
+  supportedParameters: string[]
+  inputCost: number
+  outputCost: number
+  throughput?: number // Optional based on examples
+  latency?: number // Optional based on examples
+}
+
+// Represents the nested 'features.supportedParameters' object
+interface SupportedParametersFeatures {
+  responseFormat?: boolean
+  structuredOutputs?: boolean
+}
+
+// Base Model structure (common fields without nested endpoint/providers)
+// This helps avoid circular type definitions initially
+interface ModelBase {
+  slug: string
+  hfSlug: string | null
+  updatedAt: string
+  createdAt: string
+  hfUpdatedAt: string | null // Assuming string like others, though only null seen
+  name: string
+  shortName: string
+  author: string
+  description: string
+  modelVersionGroupId: string | null
+  contextLength: number
+  inputModalities: string[]
+  outputModalities: string[]
+  hasTextOutput: boolean
+  group: string
+  instructType: string | null
+  defaultSystem: string | null // Assuming string, though only null seen
+  defaultStops: string[] // Use string[] as it can contain values
+  hidden: boolean
+  router: string | null // Assuming string, though only null seen
+  warningMessage: string | null
+  permaslug: string
+  reasoningConfig: ReasoningConfig | null
+}
+
+// Define Endpoint structure, referencing ModelBase for its 'model' property
+interface Endpoint {
+  id: string
+  name: string
+  contextLength: number
+  model: ModelBase // Reference the base model structure
+  modelVariantSlug: string
+  modelVariantPermaslug: string
+  providerName: string
+  providerInfo: ProviderInfo
+  providerDisplayName: string
+  providerModelId: string
+  providerGroup: string
+  isCloaked: boolean
+  quantization: string | null
+  variant: string
+  isSelfHosted: boolean
+  canAbort: boolean
+  maxPromptTokens: number | null
+  maxCompletionTokens: number | null // Endpoint might have different limits
+  maxPromptImages: number | null
+  maxTokensPerImage: number | null
+  supportedParameters: string[]
+  isByok: boolean
+  moderationRequired: boolean
+  dataPolicy: DataPolicy // Data policy specific to the endpoint variant
+  pricing: Pricing // Pricing specific to the endpoint variant
+  isHidden: boolean
+  isDeranked: boolean
+  isDisabled: boolean
+  supportsToolParameters: boolean
+  supportsReasoning: boolean
+  supportsMultipart: boolean
+  limitRpm: number | null
+  limitRpd: number | null
+  hasCompletions: boolean
+  hasChatCompletions: boolean
+  features: {
+    supportedParameters?: SupportedParametersFeatures
+  }
+  providerRegion: string | null // Assuming string, though only null seen
+}
+
+// --- Final Composed Model Interface ---
+
+// Extend ModelBase and add the nested structures
+interface Model extends ModelBase {
+  endpoint: Endpoint | null // Endpoint can be null
+  sorting: Sorting
+  providers: ModelProvider[] // Array can be empty
+}
+
+// --- The Default Export ---
+
+declare const _default: {
+  models: Model[]
+}
+
+export default _default