@hasna/connectors 1.3.2 → 1.3.5

package/connectors/connect-huggingface/src/api/index.ts

@@ -1,6 +1,9 @@
 import type { HuggingFaceConfig } from '../types';
 import { HuggingFaceClient } from './client';
-import { ExampleApi } from './example';
+import { ModelsApi } from './models';
+import { InferenceApi } from './inference';
+import { DatasetsApi } from './datasets';
+import { SpacesApi } from './spaces';
 
 /**
  * Main HuggingFace API class
@@ -8,42 +11,37 @@ import { ExampleApi } from './example';
 export class HuggingFace {
   private readonly client: HuggingFaceClient;
 
-  // API modules - add more as needed
-  public readonly example: ExampleApi;
+  public readonly models: ModelsApi;
+  public readonly inference: InferenceApi;
+  public readonly datasets: DatasetsApi;
+  public readonly spaces: SpacesApi;
 
   constructor(config: HuggingFaceConfig) {
     this.client = new HuggingFaceClient(config);
-    this.example = new ExampleApi(this.client);
+    this.models = new ModelsApi(this.client);
+    this.inference = new InferenceApi(this.client);
+    this.datasets = new DatasetsApi(this.client);
+    this.spaces = new SpacesApi(this.client);
   }
 
-  /**
-   * Create a client from environment variables
-   * Looks for HUGGINGFACE_API_KEY or HF_TOKEN
-   */
   static fromEnv(): HuggingFace {
     const apiKey = process.env.HUGGINGFACE_API_KEY || process.env.HF_TOKEN;
     const apiSecret = process.env.HUGGINGFACE_API_SECRET;
-
-    if (!apiKey) {
-      throw new Error('HUGGINGFACE_API_KEY or HF_TOKEN environment variable is required');
-    }
+    if (!apiKey) throw new Error('HUGGINGFACE_API_KEY or HF_TOKEN environment variable is required');
     return new HuggingFace({ apiKey, apiSecret });
   }
 
-  /**
-   * Get a preview of the API key (for debugging)
-   */
   getApiKeyPreview(): string {
     return this.client.getApiKeyPreview();
   }
 
-  /**
-   * Get the underlying client for direct API access
-   */
   getClient(): HuggingFaceClient {
     return this.client;
   }
 }
 
 export { HuggingFaceClient } from './client';
-export { ExampleApi } from './example';
+export { ModelsApi } from './models';
+export { InferenceApi } from './inference';
+export { DatasetsApi } from './datasets';
+export { SpacesApi } from './spaces';

package/connectors/connect-huggingface/src/api/inference.ts

@@ -0,0 +1,199 @@
+import type { HuggingFaceClient } from './client';
+
+const INFERENCE_URL = 'https://api-inference.huggingface.co/models';
+
+export interface TextGenerationOptions {
+  max_new_tokens?: number;
+  temperature?: number;
+  top_p?: number;
+  top_k?: number;
+  repetition_penalty?: number;
+  return_full_text?: boolean;
+  stop?: string[];
+}
+
+export interface TextGenerationResult {
+  generated_text: string;
+}
+
+export interface ChatMessage {
+  role: 'system' | 'user' | 'assistant';
+  content: string;
+}
+
+export interface ChatCompletionResult {
+  choices: Array<{
+    message: { role: string; content: string };
+    finish_reason: string;
+    index: number;
+  }>;
+  model: string;
+  usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+}
+
+export class InferenceApi {
+  constructor(private readonly client: HuggingFaceClient) {}
+
+  /** Run text generation inference */
+  async textGeneration(
+    model: string,
+    prompt: string,
+    options: TextGenerationOptions = {}
+  ): Promise<TextGenerationResult[]> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.client.getApiKey()}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        inputs: prompt,
+        parameters: {
+          max_new_tokens: options.max_new_tokens ?? 256,
+          temperature: options.temperature,
+          top_p: options.top_p,
+          top_k: options.top_k,
+          repetition_penalty: options.repetition_penalty,
+          return_full_text: options.return_full_text ?? false,
+          stop: options.stop,
+        },
+      }),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(`Inference failed (${response.status}): ${text}`);
+    }
+
+    return response.json();
+  }
+
+  /** Chat completion via HF Inference API (for chat models) */
+  async chat(
+    model: string,
+    messages: ChatMessage[],
+    options: Omit<TextGenerationOptions, 'return_full_text'> = {}
+  ): Promise<ChatCompletionResult> {
+    const response = await fetch(`${INFERENCE_URL}/${model}/v1/chat/completions`, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.client.getApiKey()}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model,
+        messages,
+        max_tokens: options.max_new_tokens ?? 256,
+        temperature: options.temperature,
+        top_p: options.top_p,
+        stop: options.stop,
+      }),
+    });
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(`Chat inference failed (${response.status}): ${text}`);
+    }
+
+    return response.json();
+  }
+
+  /** Classify text into categories */
+  async textClassification(
+    model: string,
+    text: string
+  ): Promise<Array<{ label: string; score: number }>> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: text }),
+    });
+    if (!response.ok) throw new Error(`Text classification failed (${response.status}): ${await response.text()}`);
+    const result = await response.json();
+    return Array.isArray(result[0]) ? result[0] : result;
+  }
+
+  /** Summarize text */
+  async summarization(
+    model: string,
+    text: string,
+    options: { max_length?: number; min_length?: number } = {}
+  ): Promise<string> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: text, parameters: options }),
+    });
+    if (!response.ok) throw new Error(`Summarization failed (${response.status}): ${await response.text()}`);
+    const result = await response.json() as Array<{ summary_text: string }>;
+    return result[0]?.summary_text ?? '';
+  }
+
+  /** Answer a question given a context */
+  async questionAnswering(
+    model: string,
+    question: string,
+    context: string
+  ): Promise<{ answer: string; score: number; start: number; end: number }> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: { question, context } }),
+    });
+    if (!response.ok) throw new Error(`QA failed (${response.status}): ${await response.text()}`);
+    return response.json();
+  }
+
+  /** Translate text */
+  async translation(model: string, text: string): Promise<string> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: text }),
+    });
+    if (!response.ok) throw new Error(`Translation failed (${response.status}): ${await response.text()}`);
+    const result = await response.json() as Array<{ translation_text: string }>;
+    return result[0]?.translation_text ?? '';
+  }
+
+  /** Zero-shot classification — classify without training examples */
+  async zeroShotClassification(
+    model: string,
+    text: string,
+    candidateLabels: string[]
+  ): Promise<{ sequence: string; labels: string[]; scores: number[] }> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: text, parameters: { candidate_labels: candidateLabels } }),
+    });
+    if (!response.ok) throw new Error(`Zero-shot classification failed (${response.status}): ${await response.text()}`);
+    return response.json();
+  }
+
+  /** Extract text embeddings (feature extraction) */
+  async featureExtraction(model: string, text: string | string[]): Promise<number[][]> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: text }),
+    });
+    if (!response.ok) throw new Error(`Feature extraction failed (${response.status}): ${await response.text()}`);
+    return response.json();
+  }
+
+  /** Generate an image from a text prompt */
+  async textToImage(
+    model: string,
+    prompt: string,
+    options: { negative_prompt?: string; width?: number; height?: number; num_inference_steps?: number } = {}
+  ): Promise<ArrayBuffer> {
+    const response = await fetch(`${INFERENCE_URL}/${model}`, {
+      method: 'POST',
+      headers: { Authorization: `Bearer ${this.client.getApiKey()}`, 'Content-Type': 'application/json' },
+      body: JSON.stringify({ inputs: prompt, parameters: options }),
+    });
+    if (!response.ok) throw new Error(`Text-to-image failed (${response.status}): ${await response.text()}`);
+    return response.arrayBuffer();
+  }
+}

package/connectors/connect-huggingface/src/api/models.ts

@@ -0,0 +1,66 @@
+import type { HuggingFaceClient } from './client';
+
+export interface ModelSearchOptions {
+  search?: string;
+  author?: string;
+  filter?: string;       // task filter: text-generation, text2text-generation, etc
+  library?: string;      // transformers, gguf, pytorch, etc
+  sort?: 'likes' | 'downloads' | 'trending' | 'lastModified';
+  direction?: 'asc' | 'desc';
+  limit?: number;
+  full?: boolean;        // include all fields
+}
+
+export interface ModelInfo {
+  _id: string;
+  id: string;            // e.g. "meta-llama/Meta-Llama-3-8B"
+  modelId: string;
+  author?: string;
+  sha?: string;
+  lastModified?: string;
+  private?: boolean;
+  disabled?: boolean;
+  gated?: boolean | string;
+  pipeline_tag?: string;
+  tags?: string[];
+  downloads?: number;
+  likes?: number;
+  library_name?: string;
+  [key: string]: unknown;
+}
+
+export interface ModelFile {
+  rfilename: string;
+  size?: number;
+  blobId?: string;
+  lfs?: { size: number; sha256: string; pointerSize: number };
+}
+
+export class ModelsApi {
+  constructor(private readonly client: HuggingFaceClient) {}
+
+  /** Search/list models */
+  async search(options: ModelSearchOptions = {}): Promise<ModelInfo[]> {
+    const params: Record<string, string | number | boolean | undefined> = {};
+    if (options.search) params.search = options.search;
+    if (options.author) params.author = options.author;
+    if (options.filter) params.filter = options.filter;
+    if (options.library) params.library = options.library;
+    if (options.sort) params.sort = options.sort;
+    if (options.direction) params.direction = options.direction === 'desc' ? '-1' : '1';
+    if (options.limit) params.limit = options.limit;
+    if (options.full) params.full = true;
+
+    return this.client.request<ModelInfo[]>('/models', { params });
+  }
+
+  /** Get a single model by ID (e.g. "meta-llama/Meta-Llama-3-8B") */
+  async get(modelId: string): Promise<ModelInfo> {
+    return this.client.request<ModelInfo>(`/models/${modelId}`);
+  }
+
+  /** List files in a model repo */
+  async files(modelId: string): Promise<ModelFile[]> {
+    return this.client.request<ModelFile[]>(`/models/${modelId}/tree/main`);
+  }
+}

package/connectors/connect-huggingface/src/api/spaces.ts

@@ -0,0 +1,42 @@
+import type { HuggingFaceClient } from './client';
+
+export interface SpaceSearchOptions {
+  search?: string;
+  author?: string;
+  sort?: 'likes' | 'trending' | 'lastModified';
+  direction?: 'asc' | 'desc';
+  limit?: number;
+}
+
+export interface SpaceInfo {
+  _id: string;
+  id: string;
+  author?: string;
+  sha?: string;
+  lastModified?: string;
+  private?: boolean;
+  tags?: string[];
+  likes?: number;
+  sdk?: string;
+  runtime?: { stage: string; hardware?: { current?: string } };
+  [key: string]: unknown;
+}
+
+export class SpacesApi {
+  constructor(private readonly client: HuggingFaceClient) {}
+
+  async search(options: SpaceSearchOptions = {}): Promise<SpaceInfo[]> {
+    const params: Record<string, string | number | boolean | undefined> = {};
+    if (options.search) params.search = options.search;
+    if (options.author) params.author = options.author;
+    if (options.sort) params.sort = options.sort;
+    if (options.direction) params.direction = options.direction === 'desc' ? '-1' : '1';
+    if (options.limit) params.limit = options.limit;
+
+    return this.client.request<SpaceInfo[]>('/spaces', { params });
+  }
+
+  async get(spaceId: string): Promise<SpaceInfo> {
+    return this.client.request<SpaceInfo>(`/spaces/${spaceId}`);
+  }
+}

package/connectors/connect-huggingface/src/cli/index.ts

@@ -188,55 +188,167 @@ configCmd
   });
 
 // ============================================
-// Example API Commands - Replace with HuggingFace API commands
+// Models Commands
 // ============================================
-const exampleCmd = program
-  .command('example')
-  .description('Example API commands (replace with HuggingFace commands)');
+const modelsCmd = program.command('models').description('Search and browse HuggingFace models');
 
-exampleCmd
-  .command('list')
-  .description('List resources')
-  .option('-n, --max <number>', 'Maximum results', '10')
-  .action(async (opts) => {
+modelsCmd
+  .command('search')
+  .description('Search models')
+  .argument('[query]', 'Search query')
+  .option('--task <task>', 'Filter by task (text-generation, text2text-generation, etc)')
+  .option('--library <lib>', 'Filter by library (transformers, gguf, pytorch)')
+  .option('--author <author>', 'Filter by author')
+  .option('--sort <field>', 'Sort by: likes, downloads, trending, lastModified', 'trending')
+  .option('--limit <n>', 'Max results', '20')
+  .action(async (query, opts) => {
     try {
       const client = getClient();
-      const result = await client.example.list({ maxResults: parseInt(opts.max) });
-      print(result, getFormat(exampleCmd));
-    } catch (err) {
-      error(String(err));
-      process.exit(1);
-    }
+      const results = await client.models.search({
+        search: query, filter: opts.task, library: opts.library,
+        author: opts.author, sort: opts.sort, limit: parseInt(opts.limit),
+      });
+      print(results.map(m => ({ id: m.id, task: m.pipeline_tag, library: m.library_name, downloads: m.downloads, likes: m.likes })), getFormat(modelsCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
   });
 
-exampleCmd
+modelsCmd
   .command('get <id>')
-  .description('Get a resource by ID')
-  .action(async (id: string) => {
+  .description('Get model details (e.g. meta-llama/Meta-Llama-3-8B)')
+  .action(async (id) => {
     try {
       const client = getClient();
-      const result = await client.example.get(id);
-      print(result, getFormat(exampleCmd));
-    } catch (err) {
-      error(String(err));
-      process.exit(1);
-    }
+      const result = await client.models.get(id);
+      print(result, getFormat(modelsCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
   });
 
-exampleCmd
-  .command('create')
-  .description('Create a new resource')
-  .requiredOption('-n, --name <name>', 'Resource name')
-  .action(async (opts) => {
+modelsCmd
+  .command('files <id>')
+  .description('List files in a model repo')
+  .action(async (id) => {
     try {
       const client = getClient();
-      const result = await client.example.create({ name: opts.name });
-      success('Resource created!');
-      print(result, getFormat(exampleCmd));
-    } catch (err) {
-      error(String(err));
-      process.exit(1);
-    }
+      const files = await client.models.files(id);
+      print(files.map(f => ({ name: f.rfilename, size: f.lfs?.size ?? f.size ?? null })), getFormat(modelsCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+// ============================================
+// Inference Commands
+// ============================================
+const inferCmd = program.command('inference').description('Run model inference via HF Inference API');
+
+inferCmd
+  .command('text-generation <model>')
+  .description('Generate text from a prompt')
+  .requiredOption('--prompt <text>', 'Input prompt')
+  .option('--max-tokens <n>', 'Max new tokens', '256')
+  .option('--temperature <t>', 'Temperature', '0.7')
+  .action(async (model, opts) => {
+    try {
+      const client = getClient();
+      const results = await client.inference.textGeneration(model, opts.prompt, {
+        max_new_tokens: parseInt(opts.maxTokens), temperature: parseFloat(opts.temperature),
+      });
+      print(results, getFormat(inferCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+inferCmd
+  .command('chat <model>')
+  .description('Chat completion (for chat models)')
+  .requiredOption('--messages <json>', 'Messages JSON array')
+  .option('--max-tokens <n>', 'Max new tokens', '256')
+  .option('--temperature <t>', 'Temperature', '0.7')
+  .action(async (model, opts) => {
+    try {
+      const messages = JSON.parse(opts.messages);
+      const client = getClient();
+      const result = await client.inference.chat(model, messages, {
+        max_new_tokens: parseInt(opts.maxTokens), temperature: parseFloat(opts.temperature),
+      });
+      print(result, getFormat(inferCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+// ============================================
+// Datasets Commands
+// ============================================
+const datasetsCmd = program.command('datasets').description('Search and browse HuggingFace datasets');
+
+datasetsCmd
+  .command('search')
+  .description('Search datasets')
+  .argument('[query]', 'Search query')
+  .option('--author <author>', 'Filter by author')
+  .option('--sort <field>', 'Sort by: likes, downloads, trending', 'trending')
+  .option('--limit <n>', 'Max results', '20')
+  .action(async (query, opts) => {
+    try {
+      const client = getClient();
+      const results = await client.datasets.search({
+        search: query, author: opts.author, sort: opts.sort, limit: parseInt(opts.limit),
+      });
+      print(results.map(d => ({ id: d.id, downloads: d.downloads, likes: d.likes, tags: d.tags?.slice(0, 5) })), getFormat(datasetsCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+datasetsCmd
+  .command('get <id>')
+  .description('Get dataset details')
+  .action(async (id) => {
+    try {
+      const client = getClient();
+      const result = await client.datasets.get(id);
+      print(result, getFormat(datasetsCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+datasetsCmd
+  .command('preview <id>')
+  .description('Preview first N rows of a dataset')
+  .option('--split <split>', 'Dataset split', 'train')
+  .option('--rows <n>', 'Number of rows', '10')
+  .action(async (id, opts) => {
+    try {
+      const client = getClient();
+      const result = await client.datasets.preview(id, 'default', opts.split, parseInt(opts.rows));
+      print(result, getFormat(datasetsCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+// ============================================
+// Spaces Commands
+// ============================================
+const spacesCmd = program.command('spaces').description('Search and browse HuggingFace Spaces');
+
+spacesCmd
+  .command('search')
+  .description('Search spaces')
+  .argument('[query]', 'Search query')
+  .option('--author <author>', 'Filter by author')
+  .option('--sort <field>', 'Sort by: likes, trending', 'trending')
+  .option('--limit <n>', 'Max results', '20')
+  .action(async (query, opts) => {
+    try {
+      const client = getClient();
+      const results = await client.spaces.search({
+        search: query, author: opts.author, sort: opts.sort, limit: parseInt(opts.limit),
+      });
+      print(results.map(s => ({ id: s.id, sdk: s.sdk, likes: s.likes })), getFormat(spacesCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
+  });
+
+spacesCmd
+  .command('get <id>')
+  .description('Get space details')
+  .action(async (id) => {
+    try {
+      const client = getClient();
+      const result = await client.spaces.get(id);
+      print(result, getFormat(spacesCmd));
+    } catch (err) { error(String(err)); process.exit(1); }
   });
 
 // Parse and execute