@dxos/plugin-transformer 0.7.5-main.b19bfc8

package/src/hooks/usePipeline.ts

@@ -0,0 +1,153 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Pipeline, pipeline, env } from '@xenova/transformers';
+import { useState, useRef, useEffect } from 'react';
+
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+// Add WebGPU types.
+declare global {
+  interface Navigator {
+    gpu?: {
+      requestAdapter(): Promise<GPUAdapter | null>;
+    };
+  }
+
+  interface GPUAdapter {
+    requestAdapterInfo(): Promise<GPUAdapterInfo>;
+  }
+
+  interface GPUAdapterInfo {
+    vendor: string;
+    architecture: string;
+    description: string;
+  }
+}
+
+// Configure cache and runtime settings.
+env.cacheDir = './.cache';
+env.allowLocalModels = true;
+
+// Configure ONNX runtime for WebGPU.
+(env.backends.onnx as any).wasm.numThreads = 1;
+(env.backends.onnx as any).provider = 'webgpu';
+(env.backends.onnx as any).webgpu = {
+  profilingMode: true,
+};
+
+export type PipelineConfig = {
+  active?: boolean;
+  debug?: boolean;
+  model: string;
+};
+
+export type PipelineState = {
+  gpuInfo: string;
+  isLoaded: boolean;
+  isLoading: boolean;
+  error: string | null;
+};
+
+// TODO(burdon): Document external API.
+export type TranscriptionOptions = {
+  sampling_rate: number;
+  chunk_length_s: number;
+  stride_length_s: number;
+  return_timestamps: boolean;
+  language: string;
+};
+
+export const usePipeline = ({ active, model, debug }: PipelineConfig) => {
+  const [state, setState] = useState<PipelineState>({
+    gpuInfo: '',
+    isLoaded: false,
+    isLoading: false,
+    error: null,
+  });
+
+  const pipelineRef = useRef<Pipeline | null>(null);
+
+  // TODO(burdon): Factor out loading model. Separate tests.
+  useEffect(() => {
+    const loadModel = async () => {
+      try {
+        setState((prev) => ({ ...prev, isLoading: true, error: null }));
+
+        // Check WebGPU support.
+        if (!navigator.gpu) {
+          log.warn('WebGPU is not supported, falling back to CPU');
+          setState((prev) => ({ ...prev, gpuInfo: 'WebGPU not supported (using CPU)' }));
+        } else {
+          try {
+            const adapter = await navigator.gpu.requestAdapter();
+            if (!adapter) {
+              throw new Error('No GPU adapter found');
+            }
+
+            // Try to get adapter info if available.
+            try {
+              const adapterInfo = await adapter.requestAdapterInfo();
+              if (adapterInfo) {
+                setState((prev) => ({
+                  ...prev,
+                  gpuInfo: `${adapterInfo.description || 'GPU'} (${adapterInfo.vendor || 'Unknown'})`,
+                }));
+              } else {
+                setState((prev) => ({ ...prev, gpuInfo: 'GPU Available (details unknown)' }));
+              }
+            } catch (err) {
+              log.warn('could not get GPU info', { err });
+              setState((prev) => ({ ...prev, gpuInfo: 'GPU Available (details unavailable)' }));
+            }
+          } catch (err) {
+            log.warn('WebGPU initialization failed', { err });
+            setState((prev) => ({ ...prev, gpuInfo: 'GPU initialization failed (using CPU)' }));
+          }
+        }
+
+        const pipe = await pipeline('automatic-speech-recognition', model, {
+          quantized: true,
+          progress_callback: (progress: any) => {
+            if (debug) {
+              log(`loading model: ${Math.round(progress.progress * 100)}%`);
+            }
+          },
+        });
+
+        pipelineRef.current = pipe;
+        setState((prev) => ({ ...prev, isLoaded: true, isLoading: false }));
+        log.info('model loaded successfully');
+      } catch (err) {
+        log.error('error loading model', { err });
+        setState((prev) => ({
+          ...prev,
+          isLoading: false,
+          error: 'error loading model: ' + (err as Error).message,
+        }));
+      }
+    };
+
+    if (active) {
+      void loadModel();
+    }
+
+    return () => {
+      pipelineRef.current = null;
+      setState((prev) => ({ ...prev, isLoaded: false }));
+    };
+  }, [active, debug, model]);
+
+  const transcribe = async (audioData: Float32Array, options: TranscriptionOptions) => {
+    invariant(pipelineRef.current, 'pipeline not initialized');
+    const result = await pipelineRef.current(audioData, options);
+    return result;
+  };
+
+  return {
+    ...state,
+    transcribe,
+  };
+};

package/src/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './meta';
+
+export * from './TransformerPlugin';

package/src/meta.ts

@@ -0,0 +1,16 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type PluginMeta } from '@dxos/app-framework';
+
+export const TRANSFORMER_PLUGIN = 'dxos.org/plugin/transformer';
+
+export const meta = {
+  id: TRANSFORMER_PLUGIN,
+  name: 'Transformer',
+  description: 'Run local transformers.',
+  icon: 'ph--cpu--regular',
+  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/experimental/plugin-transformer',
+  tags: ['experimental'],
+} satisfies PluginMeta;

package/src/testing/model.test.ts

@@ -0,0 +1,3 @@
+//
+// Copyright 2025 DXOS.org
+//

package/src/testing/node-pipeline.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { pipeline as xenovaPipeline } from '@xenova/transformers';
+
+import { type EmbeddingOutput, RagPipeline } from './pipeline';
+
+/**
+ * Node implementation using Xenova transformers.
+ */
+export class NodeRagPipeline extends RagPipeline {
+  private embeddingModel: string;
+  private textModel: string;
+
+  constructor(embeddingModel = 'Xenova/all-MiniLM-L6-v2', textModel = 'Xenova/t5-small') {
+    super();
+    this.embeddingModel = embeddingModel;
+    this.textModel = textModel;
+  }
+
+  protected async generateEmbeddings(text: string | string[]): Promise<EmbeddingOutput> {
+    const embedding = await xenovaPipeline('feature-extraction', this.embeddingModel);
+    const result = await embedding(text, { pooling: 'mean', normalize: true });
+    return { data: Array.from(result.data) };
+  }
+
+  protected async generateText(prompt: string) {
+    const generator = await xenovaPipeline('text2text-generation', this.textModel);
+    return generator(prompt, {
+      num_return_sequences: 1,
+      max_new_tokens: 50,
+    });
+  }
+}

package/src/testing/pipeline.test.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { AutoTokenizer, pipeline } from '@huggingface/transformers';
+import { describe, test } from 'vitest';
+
+import { log } from '@dxos/log';
+
+import { NodeRagPipeline } from './node-pipeline';
+
+describe('transformers', () => {
+  test.skip('tokenizer', async () => {
+    const tokenizer = await AutoTokenizer.from_pretrained('Xenova/bert-base-uncased');
+    const tokens = await tokenizer('I love transformers!');
+    log.info('tokens', { tokens });
+  });
+
+  test.skip('sentiment', async ({ expect }) => {
+    const sentiment = await pipeline('sentiment-analysis');
+    const result = await sentiment('I love transformers!');
+    expect(result[0]).to.include({ label: 'POSITIVE' });
+    log.info('result', { result });
+  }, 30_000);
+
+  test.skip('run embeddings', async ({ expect }) => {
+    const content = [
+      'create map of london',
+      'create function to generate the fibonacci sequence',
+      'create a table with',
+    ];
+
+    const embedding = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
+    const output = await embedding(content.join('\n'), { pooling: 'mean', normalize: true });
+    log.info('output', { output });
+  });
+
+  test.skip('generation', async ({ expect }) => {
+    const generator = await pipeline('text-generation', 'Xenova/gpt2');
+    const output = await generator('create');
+    log.info('output', { output });
+  });
+});
+
+/**
+ * Note: The RAG pipeline is designed for browser environments with WebGPU support.
+ * The Xenova transformers library primarily targets browser environments and may not
+ * work correctly in Node.js/Vitest testing environments.
+ *
+ * To test this functionality:
+ * 1. Use in a browser environment
+ * 2. Ensure WebGPU is available
+ * 3. Consider using a test framework that supports browser environments (like Playwright or Cypress)
+ */
+describe('rag pipeline', () => {
+  const knowledgeBase = [
+    //
+    'create a new table.',
+    'create a new kanban.',
+    'create a map of london.',
+    'create a function to generate the fibonacci sequence.',
+  ];
+
+  test.skip('prediction pipeline', async ({ expect }) => {
+    const pipeline = new NodeRagPipeline();
+    const result = await pipeline.generateCompletions('create a function', knowledgeBase);
+
+    expect(result).toBeDefined();
+    expect(Array.isArray(result)).toBe(true);
+    expect(result.length).toBeGreaterThan(0);
+
+    log.info('result', { result });
+  }, 120_000);
+
+  test('semantic similarity with local pipeline', async ({ expect }) => {
+    const semanticKnowledgeBase = [
+      'The quick brown fox jumps over the lazy dog.',
+      'JavaScript is a programming language.',
+      'Python is used for data science.',
+    ];
+
+    const pipeline = new NodeRagPipeline();
+    const result = await pipeline.generateCompletions('What can I use for coding?', semanticKnowledgeBase);
+
+    expect(result).toBeDefined();
+    expect(Array.isArray(result)).toBe(true);
+
+    log.info('result', { result });
+  }, 120_000);
+});

package/src/testing/pipeline.ts

@@ -0,0 +1,74 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { log } from '@dxos/log';
+
+/**
+ * Types for embeddings and model outputs.
+ */
+export type EmbeddingOutput = {
+  data: number[]; // Simplified to number[] since we'll convert all arrays.
+};
+
+/**
+ * Base class for RAG pipeline implementations.
+ */
+export abstract class RagPipeline {
+  /**
+   * Generate embeddings for a text or array of texts.
+   */
+  protected abstract generateEmbeddings(text: string | string[]): Promise<EmbeddingOutput>;
+
+  /**
+   * Generate text completions given a prompt.
+   */
+  protected abstract generateText(prompt: string): Promise<any>;
+
+  /**
+   * Calculate cosine similarity between two vectors.
+   */
+  protected cosineSimilarity(a: Float32Array | number[], b: Float32Array | number[]): number {
+    const aArray = Array.from(a);
+    const bArray = Array.from(b);
+    const dotProduct = aArray.reduce((sum, val, i) => sum + val * bArray[i], 0);
+    const magnitudeA = Math.sqrt(aArray.reduce((sum, val) => sum + val * val, 0));
+    const magnitudeB = Math.sqrt(bArray.reduce((sum, val) => sum + val * val, 0));
+    return dotProduct / (magnitudeA * magnitudeB);
+  }
+
+  /**
+   * Generate completions using the RAG pipeline.
+   */
+  async generateCompletions(input: string, knowledgeBase: string[]) {
+    // Generate embeddings for input and knowledge base.
+    const inputEmbedding = await this.generateEmbeddings(input);
+    const knowledgeEmbeddings = await Promise.all(knowledgeBase.map((text) => this.generateEmbeddings(text)));
+
+    // Calculate similarity scores.
+    const similarities = knowledgeEmbeddings.map((emb) => this.cosineSimilarity(inputEmbedding.data, emb.data));
+
+    // Get top 3 most relevant contexts.
+    const topContexts = similarities
+      .map((score, idx) => ({ score, text: knowledgeBase[idx] }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, 3)
+      .map((item) => item.text);
+
+    log.info('topContexts', { topContexts });
+
+    // Generate completions using retrieved context.
+    // const prompt = `
+    // Given the history and list of commands below, complete the following user input.
+    // History: ''
+    // Context: ${topContexts.join('\n')}
+    // User input: "${input}"
+    // `.trim();
+
+    const prompt = 'hello';
+
+    // `Context: ${topContexts.join('\n')}\nQuery: ${input}\nResponse:`;
+    log.info('prompt', { prompt });
+    return this.generateText(prompt);
+  }
+}

package/src/testing/web-pipeline.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { pipeline } from '@huggingface/transformers';
+
+import { type EmbeddingOutput, RagPipeline } from './pipeline';
+
+// TODO(burdon): Workers.
+//  https://huggingface.co/docs/transformers.js/tutorials/react#step-3-design-the-user-interface
+// TODO(burdon): WebGPT.
+//  https://huggingface.co/docs/transformers.js/guides/webgpu
+
+/**
+ * Web-based implementation using Hugging Face transformers.
+ */
+export class WebRagPipeline extends RagPipeline {
+  private embeddingModel: string;
+  private textModel: string;
+
+  constructor(embeddingModel = 'Xenova/all-MiniLM-L6-v2', textModel = 'Xenova/distilgpt2') {
+    super();
+    this.embeddingModel = embeddingModel;
+    this.textModel = textModel;
+  }
+
+  protected async generateEmbeddings(text: string | string[]): Promise<EmbeddingOutput> {
+    const embedding = await pipeline('feature-extraction', this.embeddingModel, {
+      revision: 'main',
+    });
+    const result = await embedding(text, { pooling: 'mean', normalize: true });
+    // Convert DataArray to number[]
+    return { data: Array.from(result.data) };
+  }
+
+  protected async generateText(prompt: string) {
+    const generator = await pipeline('text-generation', this.textModel, {
+      revision: 'main',
+    });
+    return generator(prompt, {
+      max_new_tokens: 50,
+      num_return_sequences: 3,
+      temperature: 0.7,
+    });
+  }
+}

package/src/translations.ts

@@ -0,0 +1,15 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { TRANSFORMER_PLUGIN } from './meta';
+
+export default [
+  {
+    'en-US': {
+      [TRANSFORMER_PLUGIN]: {
+        'plugin name': 'Transformers',
+      },
+    },
+  },
+];

package/src/types/index.ts

@@ -0,0 +1,3 @@
+//
+// Copyright 2025 DXOS.org
+//