@meaningfully/ui 0.0.1

package/dist/components/DatabaseConfig.svelte

@@ -0,0 +1,485 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import { navigate } from 'svelte-routing';
+  import { debounce } from 'lodash';
+  import Preview from './Preview.svelte';
+  import { fileDataStore } from '../stores/fileDataStore';
+  import type { MeaningfullyAPI } from '../types';
+
+  interface Props {
+    validApiKeysSet: boolean;
+    api: MeaningfullyAPI;
+  };
+
+  let {
+    validApiKeysSet,
+    api
+  } = $props();
+
+  let fileData: any = $state(null);
+  let uploading = $state(false);
+  let error = $state('');
+  let selectedTextColumn = $state('');
+  let selectedMetadataColumns: string[] = $state([]);
+  let generatingPreview = $state(false);
+  let datasetName = $state('');
+  const defaultChunkSize = 100;
+  const defaultChunkOverlap = 20;
+  let chunkSize = $state(defaultChunkSize);
+  let chunkOverlap = $state(defaultChunkOverlap);
+  
+  // Model options grouped by provider
+  const modelOptions: Record<string, string[]> = {
+    "openai": ["text-embedding-3-small", "text-embedding-3-large"],
+    "azure": ["text-embedding-3-small", "text-embedding-3-large"],
+    "ollama": ["mxbai-embed-large", "nomic-embed-text"],
+    "mistral": ["mistral-embed"],
+    "gemini": ["gemini-embedding-001"]
+  };
+  
+  let modelProvider = $state("openai");
+  let modelName = $state("text-embedding-3-small");
+  let splitIntoSentences = $state(true);
+  let combineSentencesIntoChunks = $state(true);
+  let previewData: Array<Record<string, any>> = $state([]);
+  let costEstimate: number = $state(0);
+  let tokenCount: number = $state(0);
+  let pricePer1M: number = $state(0);
+  let isCollapsed = $state(true);
+  
+  // Update model name when provider changes
+  $effect(() => {
+    if (modelProvider && modelOptions[modelProvider]) {
+      modelName = modelOptions[modelProvider][0];
+    }
+  });
+
+  // Progress tracking
+  let progress = $state(0);
+  let progressTotal = $state(100);
+  // let elapsedTimeMs = $state(0);
+  let estimatedTimeRemainingMs: number | null = $state(null);
+
+  // Helper function to format time in human-readable format
+  const formatTime = (timeMs: number | null): string => {
+    if (!timeMs || timeMs < 1000) return '<1s';
+    
+    const totalSeconds = Math.floor(timeMs / 1000);
+    const hours = Math.floor(totalSeconds / 3600);
+    const minutes = Math.floor((totalSeconds % 3600) / 60);
+    const seconds = totalSeconds % 60;
+    
+    let result = '';
+    if (hours > 0) result += `${hours}h`;
+    if (minutes > 0) result += `${minutes}m`;
+    if (seconds > 0 || result === '') result += `${seconds}s`;
+    
+    return result;
+  };
+
+  onMount(() => {
+    // Subscribe to the file data store
+    const unsubscribe = fileDataStore.subscribe((data) => {
+      if (!data) {
+        navigate('/'); // Redirect back to home if no file data
+        return;
+      }
+      fileData = data;
+      datasetName = fileData.name.replace(/\.csv$/, '');
+    });
+
+    // Ensure cleanup on destroy
+    // https://svelte.dev/docs/svelte/lifecycle-hooks
+    // "If a function is returned from onMount, it will be called when the component is unmounted."
+    return unsubscribe;
+  });
+
+  // Poll the backend every second for upload progress
+  const pollProgress = async () => {
+    if (!uploading) return;
+    try {
+      const result = await api.getUploadProgress();
+      progress = result.progress;
+      progressTotal = result.total;
+      // // elapsedTimeMs = result.elapsedTimeMs;
+      estimatedTimeRemainingMs = result.estimatedTimeRemainingMs;
+    } catch(e) {
+      console.error("Error fetching progress:", e);
+    }
+    if (uploading) setTimeout(pollProgress, 1000);
+  };
+
+  const generatePreview = async () => {
+    if (!fileData || !selectedTextColumn) {
+      error = 'Select a text column (or maybe something went wrong with the file you chose)';
+      return;
+    }
+    
+    try {
+      error = '';
+      generatingPreview = true;
+      const previewResponse = await api.generatePreviewData({
+        fileContent: fileData.fileContent,
+        fileName: fileData.name,
+        datasetName,
+        description: 'TK',
+        textColumns: [selectedTextColumn],
+        metadataColumns: selectedMetadataColumns.map(c => c),
+        splitIntoSentences,
+        combineSentencesIntoChunks,
+        sploderMaxSize: 100,
+        modelName,
+        modelProvider,
+        chunkSize,
+        chunkOverlap
+      });
+
+      if (previewResponse.success) {
+        costEstimate = previewResponse.estimatedPrice;
+        tokenCount = previewResponse.tokenCount;
+        pricePer1M = previewResponse.pricePer1M;
+        previewData = previewResponse.nodes.map((result: Record<string, any>) => ({
+          ...result.metadata,
+          [selectedTextColumn]: result.text
+        }));
+      } else {
+        error = 'Preview generation failed';
+        console.error('Preview generation failed:', previewResponse);
+      }
+    } catch (e: any) {
+      error = e.message;
+    } finally {
+      generatingPreview = false;
+    }
+  };
+
+  const handleUpload = async () => {
+    if (!fileData || !selectedTextColumn) {
+      error = 'Select a text column (or maybe something went wrong with the file you chose)';
+      return;
+    }
+    
+    try {
+      uploading = true;
+      error = '';
+      
+      // Reset timing variables
+      // elapsedTimeMs = 0;
+      estimatedTimeRemainingMs = null;
+
+      // Start polling for progress
+      pollProgress();
+      const uploadResponse = await api.uploadCsv({
+        fileContent: fileData.fileContent,
+        fileName: fileData.name,
+        datasetName,
+        description: 'TK',
+        textColumns: [selectedTextColumn],
+        metadataColumns: selectedMetadataColumns.map(c => c),
+        splitIntoSentences,
+        combineSentencesIntoChunks,
+        sploderMaxSize: 100,
+        chunkSize,
+        chunkOverlap,
+        modelName,
+        modelProvider
+      });
+
+      if (uploadResponse.success) {
+        navigate("/search/" + uploadResponse.documentSetId);
+      } else {
+        error = 'Upload failed';
+      }
+    } catch (e: any) {
+      error = e.message;
+    } finally {
+      uploading = false;
+      // Reset timing variables when upload finishes
+      // elapsedTimeMs = 0;
+      estimatedTimeRemainingMs = null;
+    }
+  };
+
+  const toggleMetadataColumn = (column: string) => {
+    if (column === selectedTextColumn) return;
+    const index = selectedMetadataColumns.indexOf(column);
+    if (index === -1) {
+      selectedMetadataColumns = [...selectedMetadataColumns, column];
+    } else {
+      selectedMetadataColumns = selectedMetadataColumns.filter(c => c !== column);
+    }
+  };
+
+  const toggleTextHandlingSectionCollapse = () => {
+    isCollapsed = !isCollapsed;
+  };
+
+  const debouncedGeneratePreview = debounce(generatePreview, 1000);
+
+  $effect(() => {
+    // Synchronously read all values to establish dependencies
+    const params = {
+      chunkSize,
+      chunkOverlap,
+      splitIntoSentences,
+      combineSentencesIntoChunks,
+      selectedTextColumn,
+      selectedMetadataColumns,
+      modelName,
+      modelProvider,
+    };
+
+    // Only trigger preview if we have required values
+    if (params.selectedTextColumn || params.selectedMetadataColumns.length) {
+      debouncedGeneratePreview();
+    }
+  });
+
+  const goBack = () => {
+    navigate('/');
+  };
+</script>
+
+{#if fileData}
+  <div class="bg-white p-6 rounded-lg shadow space-y-6 text-black mb-10">
+    <div class="flex items-center justify-between">
+      <h2 class="text-xl font-semibold">The Details: Set up semantic search for "{fileData.name}"</h2>
+      <button
+        onclick={goBack}
+        class="text-sm text-gray-600 hover:text-gray-800 underline"
+      >
+        ← Choose A Different File
+      </button>
+    </div>
+    
+    <div class="text-sm text-gray-600">
+      File size: {Math.round(fileData.size / 1024)} KB • 
+      {fileData.availableColumns.length} columns detected
+    </div>
+  </div>
+
+  <div data-testid="csv-upload-settings">
+    <div class="bg-white p-6 rounded-lg shadow space-y-6 text-black mb-10">
+      <div class="space-y-2">
+        <label class="block text-sm font-medium text-gray-700">
+          Give the spreadsheet a name:
+          <input
+            type="text"
+            data-testid="dataset-name-input"
+            bind:value={datasetName}
+            class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+          />
+        </label>
+        <p class="text-xs text-gray-500">
+          The name is just for you. Use something that will help you remember what this spreadsheet is.
+        </p>
+      </div>
+      
+      <div class="space-y-2">
+        <label class="block text-sm font-medium text-gray-700">
+          What embedding provider should we use?
+          <select
+            bind:value={modelProvider}
+            class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-violet-500 focus:ring-violet-500">
+            <option value="openai">OpenAI</option>
+            <option value="azure">Azure OpenAI</option>
+            <option value="ollama">Ollama</option>
+            <option value="mistral">Mistral AI</option>
+            <option value="gemini">Google Gemini</option>
+          </select>
+        </label>  
+        <p class="text-xs text-gray-500">
+          The provider for the embedding model.
+        </p>
+      </div>
+      
+      <div class="space-y-2">
+        <label class="block text-sm font-medium text-gray-700">
+          What embedding model should we use?
+          <select
+            bind:value={modelName}
+            class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-violet-500 focus:ring-violet-500">
+            {#each modelOptions[modelProvider] as modelNameChoice}
+              <option value={modelNameChoice}>{modelNameChoice}</option>
+            {/each}
+          </select>
+        </label>  
+        <p class="text-xs text-gray-500">
+          The model used to generate embeddings for the text.
+        </p>
+      </div>
+    </div>
+  
+    <div class="bg-white p-6 rounded-lg shadow space-y-6 text-black mb-10">
+      <h3>Column Configuration</h3>
+      <div class="space-y-4">
+        <div class="space-y-2">
+          <label class="block text-sm font-medium text-gray-700">
+            Which column holds the text you want to search?
+            <select
+              bind:value={selectedTextColumn}
+              class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+              data-testid="column-to-embed-select"
+            >
+              <option value="">Select a column...</option>
+              {#each fileData.availableColumns as column}
+                <option value={column}>{column}</option>
+              {/each}
+            </select>
+          </label>
+        </div>
+
+        <div class="space-y-2">
+          <p class="block text-sm font-medium text-gray-700">
+            Which other columns should be shown in the results, and available for filtering?
+          </p>
+          <p class="text-xs text-gray-500">
+            For instance, if your spreadsheet has a <code>Category</code> column, you might want to select it so you can filter by it when searching. If it has a 
+            <code>URL</code>, you might select it so you can click through to the original.
+          </p>
+          <div class="flex flex-wrap gap-2">
+            {#each fileData.availableColumns as column}
+              <label class="inline-flex items-center">
+                <input
+                  type="checkbox"
+                  id={`metadata-${column}`}
+                  checked={selectedMetadataColumns.includes(column)}
+                  disabled={column === selectedTextColumn}
+                  onchange={() => toggleMetadataColumn(column)}
+                  class="rounded border-gray-300 text-violet-600 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+                />
+                <span class="ml-2 text-sm text-gray-700">{column}</span>
+              </label>
+            {/each}
+          </div>
+        </div>
+      </div>    
+    </div>
+    
+    <div class="bg-white p-6 rounded-lg shadow space-y-6 text-black mb-10">
+      <h3><button onclick={toggleTextHandlingSectionCollapse}>Text Handling Options {isCollapsed ? '›' : '⌄'}</button></h3>
+
+      {#if !isCollapsed}
+        <div class="space-y-6">
+          <div class="space-y-2 flex flex-wrap gap-2">
+            <p>Each text might contain multiple ideas. Meaningfully tries to represent these ideas at multiple levels, returning search results of a 1, 2 or 3 sentences long</p>
+            <label class="inline-flex items-center">
+              <input
+                type="checkbox"
+                bind:checked={splitIntoSentences}
+                class="rounded border-gray-300 text-violet-600 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+              />
+              <span class="ml-2 text-sm text-gray-700">Split into sentences?</span>
+            </label>
+            <label class="inline-flex items-center">
+              <input
+                type="checkbox"
+                bind:checked={combineSentencesIntoChunks}
+                class="rounded border-gray-300 text-violet-600 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+              />
+              <span class="ml-2 text-sm text-gray-700">Combine sentences into chunks?</span>
+            </label>
+          </div>
+
+          <div class="space-y-2">
+            <h3>Split long sentences into chunks</h3>
+            <div class="flex flex-wrap gap-2">
+              <div class="inline-flex max-w-md p-2">
+                <label class="block text-sm font-medium text-gray-700">
+                  Chunk size:
+                  <input
+                    type="number"
+                    bind:value={chunkSize}
+                    min="50"
+                    max="1000"
+                    class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+                  />
+                </label>
+              </div>
+              <div class="inline-flex max-w-md p-2">
+                <label class="block text-sm font-medium text-gray-700">
+                  Chunk overlap:
+                  <input
+                    type="number"
+                    bind:value={chunkOverlap}
+                    min="0"
+                    max="100"
+                    class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-violet-500 focus:ring-violet-500"
+                  />
+                </label>
+              </div>
+            </div>
+          </div>
+        </div>
+      {/if}
+    </div>
+    
+    <div class="bg-white p-6 rounded-lg shadow space-y-6 text-black mb-10">
+      {#key costEstimate}
+        {#key tokenCount}
+          {#key pricePer1M }
+            {#if tokenCount > 0 && costEstimate > 0}
+              <div data-testid="cost-estimate">
+                <h3>Cost Estimate</h3>
+                <p class="text-sm text-gray-600">
+                  Estimated cost: <strong>${costEstimate.toFixed(4)}</strong> 
+                  ({tokenCount.toLocaleString()} tokens at ${pricePer1M !== undefined ? pricePer1M.toFixed(2) : 'N/A'} per 1M tokens)
+                </p>
+              </div>
+            {/if}
+          {/key}
+        {/key}
+      {/key}
+      
+      {#key previewData}
+        {#key selectedTextColumn}
+          {#key selectedMetadataColumns}
+            {#if (previewData.length > 0 || generatingPreview) && selectedTextColumn}
+              <Preview
+                previewData={previewData}
+                textColumn={selectedTextColumn}
+                metadataColumns={selectedMetadataColumns}
+                loading={generatingPreview}
+              />
+            {/if}
+          {/key}
+        {/key}
+      {/key}
+
+      <button
+        onclick={handleUpload}
+        data-testid="upload-button"
+        disabled={!selectedTextColumn || uploading || !validApiKeysSet}
+        class="inline-flex justify-center py-2 px-4 border border-transparent shadow-sm text-sm font-medium rounded-md text-white bg-violet-600 hover:bg-violet-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-violet-500 disabled:opacity-50 disabled:cursor-not-allowed"
+      >
+        {uploading ? 'Uploading...' : 'Upload Spreadsheet'}
+      </button>
+      
+      {#if uploading}
+        <p>This could take a few minutes. Go get a cup of coffee or re-arrange your MySpace Top 8.</p>
+        <div class="w-full bg-gray-200 rounded-full h-4 mt-2">
+          <div 
+            class="bg-violet-600 h-4 rounded-full transition-all duration-500" 
+            style="width: {Math.min(100, Math.round((progress / progressTotal) * 100))}%"
+          ></div>
+        </div>
+        <div class="text-sm text-gray-600 mt-2 space-y-1">
+          <p>
+            Progress: {progress <= 5 ? 'processing upload' : ( progress >= 95 ? 'finishing up' : 'embedding') }  ({Math.round((progress / progressTotal) * 100)}%)
+            {#if estimatedTimeRemainingMs !== null}
+              | {formatTime(estimatedTimeRemainingMs)} remaining (est.)
+            {/if}
+          </p>
+        </div>
+      {/if }
+    </div>
+  </div>
+{:else}
+  <div class="bg-white p-6 rounded-lg shadow space-y-6 text-black mb-10">
+    <p>Loading file data...</p>
+  </div>
+{/if}
+
+{#if error}
+  <div class="mt-4 text-red-600">{error}</div>
+{/if}

package/dist/components/DatabaseConfig.svelte.d.ts

@@ -0,0 +1,6 @@
+declare const DatabaseConfig: import("svelte").Component<{
+    validApiKeysSet: any;
+    api: any;
+}, {}, "">;
+type DatabaseConfig = ReturnType<typeof DatabaseConfig>;
+export default DatabaseConfig;

package/dist/components/ExistingDatabases.svelte

@@ -0,0 +1,175 @@
+<script lang="ts">
+  import { onMount } from 'svelte';
+  import type { DocumentSet, MeaningfullyAPI } from '../types.js';
+  import { Link } from 'svelte-routing';
+
+  interface Props {
+    api: MeaningfullyAPI;
+  }
+  let { api }: Props = $props();
+
+  let documentSets: DocumentSet[] = $state([]);
+  let loading = $state(true);
+  let error: string | null = $state(null);
+  let hidden = $state(false);
+  let currentPage = $state(1);
+  let totalPages = $state(1);
+  let totalDocuments = $state(0);
+  const pageSize = 10;
+
+  export function hide() {
+    hidden = true;
+  }
+
+  export function show() {
+    hidden = false;
+  }
+
+  export async function loadDocumentSets(page: number = 1) {    try {
+      loading = true;
+      const result = await api.listDocumentSets(page, pageSize);
+      documentSets = result.documents.map(set => ({
+        ...set,
+        uploadDate: new Date(set.uploadDate)
+      }));
+      totalDocuments = result.total;
+      totalPages = Math.ceil(totalDocuments / pageSize);
+    } catch (e) {
+      error = e instanceof Error ? e.message : 'Failed to load document sets';
+    } finally {
+      loading = false;
+    }
+  }
+
+  async function handleDelete(documentSetId: number, name: string, e: Event) {
+    e.preventDefault();
+    if (confirm(`Are you sure you want to delete "${name}"? This cannot be undone.`)) {
+      try {
+        await api.deleteDocumentSet(documentSetId);
+        await loadDocumentSets(currentPage);
+      } catch (e) {
+        error = e instanceof Error ? e.message : 'Failed to delete document set';
+      }
+    }
+  }
+
+  function nextPage(e: Event) {
+    e.preventDefault();
+    if (currentPage < totalPages) {
+      currentPage++;
+      loadDocumentSets(currentPage);
+    }
+  }
+
+  function previousPage(e: Event) {
+    e.preventDefault();
+    if (currentPage > 1) {
+      currentPage--;
+      loadDocumentSets(currentPage);
+    }
+  }
+
+ onMount(() => loadDocumentSets(1));
+</script>
+
+{#if hidden}
+  <div class="my-10 flex justify-center p-8">
+  </div>
+{:else if loading}
+  <div class="my-10 flex justify-center p-8">
+    <div class="animate-spin h-8 w-8 border-4 border-blue-500 rounded-full border-t-transparent"></div>
+  </div>
+{:else if error}
+  <div class="my-10 p-4 bg-red-100 text-red-700 rounded-md">
+    {error}
+  </div>
+{:else}
+  <div class="my-10 bg-white p-6 rounded-lg shadow space-y-6 text-black" data-testid="existing-spreadsheets">
+    <h2 class="text-2xl font-bold">Existing Spreadsheets</h2>
+    {#if documentSets.length === 0}
+      <p class="text-gray-500">No spreadsheets found. Upload one to get started.</p>
+    {:else}
+      <div class="overflow-x-auto">
+        <table class="min-w-full table-auto">
+          <thead>
+            <tr class="">
+              <th class="px-4 py-2 text-left">Name</th>
+              <th class="px-4 py-2 text-left">Upload Date</th>
+              <th class="px-4 py-2 text-left">Documents</th>
+              <th class="px-4 py-2 text-left">Parameters</th>
+              <th class="px-4 py-2 text-left"><span class="sr-only">Actions</span></th>
+            </tr>
+          </thead>
+          <tbody>
+            {#each documentSets as set}
+              <tr 
+                class="border-t hover:bg-gray-50 transition-colors" 
+                data-testid="existing-spreadsheet-row"
+              >
+                <td class="px-4 py-2 font-medium">
+                  <Link 
+                    to={`/search/${set.documentSetId}`} 
+                    class="underline text-blue-600 hover:text-blue-800 visited:text-purple-600"
+                  >
+                    {set.name}
+                  </Link>
+                </td>
+                <td class="px-4 py-2 text-gray-600">{set.uploadDate.toLocaleString()}</td>
+                <td class="px-4 py-2 text-gray-600">
+                  {set.totalDocuments.toLocaleString()}
+                </td>
+                <td class="px-4 py-2">
+                  {#if Object.keys(set.parameters).length > 0}
+                    <details>
+                      <summary class="cursor-pointer text-sm text-blue-600">View Parameters</summary>
+                      <pre class="mt-2 p-2 bg-gray-50 rounded text-sm">{JSON.stringify(set.parameters, null, 2)}</pre>
+                    </details>
+                  {:else}
+                    <span class="text-gray-400">None</span>
+                  {/if}
+                </td>
+                <td class="px-4 py-2">
+                  <button
+                    type="button"
+                    class="text-gray-500 hover:text-red-600 transition-colors"
+                    aria-label="Delete {set.name}"
+                    title="Delete {set.name}"
+                    onclick={(e) => handleDelete(set.documentSetId, set.name, e)}
+                  >
+                    <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+                      <path fill-rule="evenodd" d="M4.293 4.293a1 1 0 011.414 0L10 8.586l4.293-4.293a1 1 0 111.414 1.414L11.414 10l4.293 4.293a1 1 0 01-1.414 1.414L10 11.414l-4.293 4.293a1 1 0 01-1.414-1.414L8.586 10 4.293 5.707a1 1 0 010-1.414z" clip-rule="evenodd" />
+                    </svg>
+                  </button>
+                </td>
+              </tr>
+            {/each}
+          </tbody>
+        </table>
+        
+        {#if totalPages > 1}
+          <div class="mt-4 flex items-center justify-between px-4" data-testid="pagination-control">
+            <div class="text-sm text-gray-700">
+              Showing page {currentPage} of {totalPages}
+            </div>
+            <div class="flex gap-2">
+              <button
+                class="px-4 py-2 border rounded-md disabled:opacity-50 disabled:cursor-not-allowed"
+                onclick={previousPage}
+                disabled={currentPage === 1}
+              >
+                Previous
+              </button>
+              <button
+                class="px-4 py-2 border rounded-md disabled:opacity-50 disabled:cursor-not-allowed"
+                onclick={nextPage}
+                disabled={currentPage === totalPages}
+              >
+                Next
+              </button>
+            </div>
+          </div>
+        {/if}
+      </div>
+    {/if}
+  </div>
+{/if}

package/dist/components/ExistingDatabases.svelte.d.ts

@@ -0,0 +1,11 @@
+import type { MeaningfullyAPI } from '../types.js';
+interface Props {
+    api: MeaningfullyAPI;
+}
+declare const ExistingDatabases: import("svelte").Component<Props, {
+    hide: () => void;
+    show: () => void;
+    loadDocumentSets: (page?: number) => Promise<void>;
+}, "">;
+type ExistingDatabases = ReturnType<typeof ExistingDatabases>;
+export default ExistingDatabases;