@rebasepro/plugin-data-enhancement 0.0.1-canary.000dc36

package/src/components/FormEnhanceAction.tsx

@@ -0,0 +1,273 @@
+
+import React, { useCallback, useDeferredValue, useEffect, useRef } from "react";
+
+import { Button, CircularProgress, cls, focusedDisabled, IconButton, Menu, MenuItem, Separator, TextareaAutosize , iconSize } from "@rebasepro/ui";
+import { XIcon, SendIcon } from "lucide-react";
+import {
+    AIIcon,
+    useLargeLayout
+} from "@rebasepro/core";
+import { EntityStatus, PluginFormActionProps, Properties, Property } from "@rebasepro/types";
+import { isPropertyBuilder, stripCollectionPath } from "@rebasepro/common";
+import { useDataEnhancementController } from "./DataEnhancementControllerProvider";
+import { SamplePrompt } from "../types/data_enhancement_controller";
+
+export function FormEnhanceAction({
+    entityId,
+    path,
+    status,
+    collection,
+    formContext,
+    openEntityMode
+}: PluginFormActionProps) {
+
+    const largeLayout = useLargeLayout();
+
+    const storageKey = createLocalStorageKey(path, status);
+
+    const [loading, setLoading] = React.useState(false);
+    const dataEnhancementController = useDataEnhancementController();
+
+    const [samplePrompts, setSamplePrompts] = React.useState<SamplePrompt[] | undefined>(undefined);
+    const [instructions, setInstructions] = React.useState<string>("");
+
+    const {
+        suggestions,
+        getSamplePrompts
+    } = dataEnhancementController;
+
+    const loadingPrompts = useRef(false);
+    const updateSuggestedPrompts = useCallback(async function updateSuggestedPrompts(instructions?: string) {
+        if (loadingPrompts.current) return;
+        loadingPrompts.current = true;
+        const prompts = status === "new"
+            ? (await getSamplePrompts(collection.singularName ?? collection.name, instructions)).prompts
+            : getPromptsForExistingEntities(collection.properties);
+
+        const recentPromptsFromStorage = getRecentPromptsFromStorage(storageKey);
+        const recentPrompts = recentPromptsFromStorage.map(prompt => prompt.prompt);
+        setSamplePrompts([...recentPromptsFromStorage, ...prompts.filter(p => !recentPrompts.includes(p.prompt))].slice(0, 5));
+        loadingPrompts.current = false;
+    },
+        [collection.name, collection.singularName, getSamplePrompts, status]);
+
+    const deferredValues = useDeferredValue(formContext?.values);
+    // const enoughData = countStringCharacters(deferredValues, collection.properties) > 20;
+
+    useEffect(() => {
+        if (!samplePrompts) {
+            setSamplePrompts(getRecentPromptsFromStorage(storageKey));
+            updateSuggestedPrompts().then();
+        }
+    }, [samplePrompts, storageKey, updateSuggestedPrompts, instructions, status]);
+
+    useEffect(() => {
+        updateSuggestedPrompts().then();
+    }, [status]);
+
+    const enhance = (prompt?: string) => {
+        if (!formContext?.values) return;
+        setLoading(true);
+        if (prompt) {
+            addRecentPrompt(storageKey, prompt);
+            setSamplePrompts([{
+                prompt,
+                type: "recent"
+            }, ...(samplePrompts ?? []).slice(0, 5)]);
+        }
+        return dataEnhancementController.enhance({
+            entityId,
+            values: formContext!.values,
+            instructions: prompt,
+            replaceValues: true
+        }).finally(() => {
+            setLoading(false);
+        });
+    };
+
+    if (!dataEnhancementController?.enabled)
+        return null;
+
+    const hasSuggestions = Object.values(suggestions).filter(Boolean).length > 0;
+
+    const disabledSuggestionActions = !hasSuggestions;
+    const promptSuggestionsEnabled = (samplePrompts ?? []).length > 0 && instructions.length === 0;
+
+    // const noIdSet = !formContext?.entityId;
+
+    function submit() {
+        enhance(instructions);
+    }
+
+    return (
+        <Menu
+            align={"end"}
+            sideOffset={8}
+            className={"max-w-[100vw]"}
+            trigger={<Button variant={"filled"}
+                color={"neutral"}
+                fullWidth={largeLayout && openEntityMode === "full_screen"}
+                size={"small"}
+                disabled={loading}>
+                {!loading && <AIIcon size={"small"}/>}
+                {loading && <CircularProgress size={"small"}/>}
+                Autofill
+            </Button>}>
+
+            <MenuItem className={"py-4"}
+                onClick={() => {
+                    enhance();
+                }}>
+                <AIIcon size={"small"}/>
+                Autofill based on the current content
+            </MenuItem>
+
+            <Separator orientation={"horizontal"} className={"mt-2"}/>
+
+            {samplePrompts?.map((samplePrompt, index) => {
+                return <MenuItem
+                    key={index + "_" + samplePrompt.prompt}
+                    onClick={() => {
+                        setInstructions(samplePrompt.prompt);
+                        enhance(samplePrompt.prompt);
+                    }}
+                >
+                    <div className={"pl-9 grow text-text-secondary dark:text-text-secondary-dark"}>
+                        {samplePrompt.prompt}
+                    </div>
+
+                    {samplePrompt.type === "recent" && <IconButton
+                        onClick={(e) => {
+                            e.preventDefault();
+                            e.stopPropagation();
+                            removeRecentPrompt(storageKey, samplePrompt.prompt);
+                            setSamplePrompts((samplePrompts ?? []).filter(p => p.prompt !== samplePrompt.prompt));
+                        }}
+                        size={"smallest"}
+                    >
+                        <XIcon size={iconSize.smallest}/>
+                    </IconButton>
+                    }
+                </MenuItem>;
+            })}
+
+            <Separator orientation={"horizontal"}/>
+
+            <div
+                className={cls(
+                    "my-2 w-[500px] max-w-full flex items-start text-surface-700 dark:text-surface-200"
+                )}>
+
+                <TextareaAutosize
+                    className={cls("p-4 rounded-lg resize-none bg-surface-100 dark:bg-surface-950 mx-2 w-full grow outline-hidden max-h-[300px] overflow-auto", focusedDisabled)}
+                    value={instructions}
+                    autoFocus={status === "new"}
+                    disabled={loading}
+                    onFocus={(event) => {
+                        event.stopPropagation();
+                    }}
+                    placeholder={"...or provide instructions"}
+                    onKeyDown={(e) => {
+                        e.stopPropagation();
+                        if (e.key === "Enter" && !e.shiftKey) {
+                            e.preventDefault();
+                            submit();
+                        }
+
+                    }}
+                    onChange={(e) => {
+                        setInstructions(e.target.value);
+                    }}
+                />
+
+                <IconButton
+                    size={"small"}
+                    onClick={() => {
+                        setInstructions("");
+                    }}
+                    color={!instructions ? "primary" : undefined}
+                    disabled={loading || !instructions}>
+                    <XIcon size={iconSize.small}/>
+                </IconButton>
+
+                <IconButton
+                    onClick={() => enhance(instructions)}
+                    size={"small"}
+                    color={!instructions ? "primary" : undefined}
+                    disabled={loading || !instructions}>
+                    {loading &&
+                        <CircularProgress size={"smallest"}/>}
+                    {!loading &&
+                        <SendIcon color={"primary"}/>}
+                </IconButton>
+
+            </div>
+
+        </Menu>
+    );
+}
+
+export interface EnhanceDialogProps {
+    open: boolean;
+    onClose: () => void;
+    selectReferences: () => void;
+    loading: boolean;
+    enhance: (instructions: string) => void;
+    samplePrompts?: string[];
+}
+
+function getPromptsForExistingEntities(properties: Properties): SamplePrompt[] {
+
+    const multilineProperties = Object.values(properties).filter((p: Property) => {
+        if (isPropertyBuilder(p)) {
+            return false;
+        }
+        return p.type === "string" && (p.ui?.markdown || p.ui?.multiline);
+    });
+
+    const multilinePrompt: Property | undefined = multilineProperties.length > 0
+        ? multilineProperties[Math.floor(Math.random() * multilineProperties.length)] as Property
+        : undefined;
+
+    const prompts = [
+        "Fill the missing fields",
+        "Translate the missing content"
+    ];
+    if (multilinePrompt) {
+        prompts.push(`Add 2 paragraphs to '${multilinePrompt.name}'`);
+    }
+    return prompts.map(p => ({
+        prompt: p,
+        type: "sample"
+    }));
+}
+
+const createLocalStorageKey = (path: string, status: EntityStatus) => {
+    const statusString = status === "new" ? "new" : "existing";
+    return `data_enhancement::${statusString}::${stripCollectionPath(path)}`;
+};
+
+const getRecentPromptsFromStorage = (storageKey: string): SamplePrompt[] => {
+    const item = localStorage.getItem(storageKey);
+    return item ? JSON.parse(item).map((e: string) => ({
+        prompt: e,
+        type: "recent"
+    })) : [];
+};
+
+const addRecentPrompt = (storageKey: string, prompt: string) => {
+    if (!prompt || prompt.trim().length === 0) {
+        return;
+    }
+    const recentPrompts = getRecentPromptsFromStorage(storageKey);
+    localStorage.setItem(storageKey, JSON.stringify([prompt, ...recentPrompts
+        .map(e => e.prompt)
+        .filter(e => e !== prompt)
+        .slice(0, 5)]));
+};
+
+const removeRecentPrompt = (storageKey: string, prompt: string) => {
+    localStorage.setItem(storageKey, JSON.stringify(getRecentPromptsFromStorage(storageKey)
+        .map(e => e.prompt)
+        .filter(e => e !== prompt)));
+};

package/src/editor/useEditorAIController.tsx

@@ -0,0 +1,37 @@
+import { autocompleteStream } from "../api";
+import { EditorAIController } from "@rebasepro/admin";
+
+export function useEditorAIController({ getAuthToken }: { getAuthToken?: () => Promise<string> }): EditorAIController {
+    const autocomplete = async (textBefore: string, textAfter: string, onUpdate: (delta: string) => void) => {
+        if (!getAuthToken) {
+            throw new Error("Firebase token is required");
+        }
+        const firebaseToken = await getAuthToken();
+        return autocompleteStream({
+            firebaseToken,
+            textBefore,
+            textAfter,
+            onUpdate
+        });
+    }
+
+    return {
+        autocomplete
+    };
+}
+
+// async function * generateLoremIpsum(): AsyncGenerator<string> {
+//     const loremIpsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n# Heading\n\nLorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.";
+//
+//     const words = loremIpsum.split(" ");
+//
+//     for (const word of words) {
+//         yield word;
+//         await new Promise(resolve => setTimeout(resolve, 100));
+//     }
+// }
+//
+// const generator = generateLoremIpsum();
+// for await (const word of generator) {
+//
+// }

package/src/index.ts

@@ -0,0 +1,10 @@
+export {
+    useDataEnhancementPlugin
+} from "./useDataEnhancementPlugin";
+export type {
+    DataEnhancementPluginProps
+} from "./useDataEnhancementPlugin";
+
+export type { SubscriptionMessageProps } from "./types/subscriptions_message_props";
+
+export * from "./editor/useEditorAIController";

package/src/locales/de.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsDe = {
+    autofill: "Automatisch ausfüllen",
+    autofill_current_content: "Basierend auf dem aktuellen Inhalt automatisch ausfüllen",
+    set_id_first: "Bitte zuerst eine ID festlegen",
+    provide_instructions: "Anweisungen für die Datenerweiterung angeben",
+    fill_missing_fields: "Fehlende Felder ausfüllen",
+    translate_missing_content: "Fehlenden Inhalt übersetzen",
+    add_2_paragraphs_to: "2 Absätze zum Inhalt hinzufügen",
+    use_openai_generate_content: "Verwenden Sie Google Vertex AI oder OpenAI, um Inhalte automatisch zu generieren.",
+    autofill_this_field: "Dieses Feld automatisch ausfüllen",
+    autofill_property: "Eigenschaft automatisch ausfüllen",
+    input_data_before_enhancing: "Vor der Erweiterung Daten eingeben",
+    input_data_before_enhancing_property: "Sie müssen Daten in anderen Eigenschaften eingeben, bevor Sie diese erweitern können.",
+    based_on_rest_of_entity: "Basierend auf dem Rest der Entität",
+    this_field: "dieses Feld",
+    subscription_needed: "Für die Nutzung dieser Funktion ist ein gültiges Abonnement erforderlich.",
+    login_to_enhance: "Sie müssen angemeldet sein, um Daten zu erweitern",
+    no_fields_updated: "Keine Felder wurden aktualisiert",
+    ask_ai_to_write: "KI zum Schreiben auffordern",
+    instructions: "Anweisungen"
+};

package/src/locales/en.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsEn = {
+    autofill: "Autofill",
+    autofill_current_content: "Autofill based on current content",
+    set_id_first: "You must set an ID first",
+    provide_instructions: "Provide instructions to Data Enhancement",
+    fill_missing_fields: "Fill missing fields",
+    translate_missing_content: "Translate missing content",
+    add_2_paragraphs_to: "Add 2 paragraphs to the content",
+    use_openai_generate_content: "Use Google Vertex AI or OpenAI to generate content automatically.",
+    autofill_this_field: "Autofill this field",
+    autofill_property: "Autofill property",
+    input_data_before_enhancing: "Input some data before enhancing",
+    input_data_before_enhancing_property: "You need to input some data in other properties before enhancing this one.",
+    based_on_rest_of_entity: "Based on the rest of the entity",
+    this_field: "this field",
+    subscription_needed: "A valid subscription is needed in order to use this function.",
+    login_to_enhance: "You need to be logged in to enhance data",
+    no_fields_updated: "No fields were updated",
+    ask_ai_to_write: "Ask AI to write",
+    instructions: "Instructions"
+};

package/src/locales/es.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsEs = {
+    autofill: "Autocompletar",
+    autofill_current_content: "Autocompletar basado en el contenido actual",
+    set_id_first: "Debes establecer un ID primero",
+    provide_instructions: "Proporcionar instrucciones para Mejora de datos",
+    fill_missing_fields: "Llenar campos faltantes",
+    translate_missing_content: "Traducir contenido faltante",
+    add_2_paragraphs_to: "Añadir 2 párrafos al contenido",
+    use_openai_generate_content: "Usa Google Vertex AI u OpenAI para generar contenido automáticamente.",
+    autofill_this_field: "Autocompletar este campo",
+    autofill_property: "Propiedad de autocompletado",
+    input_data_before_enhancing: "Introduce algunos datos antes de mejorar el contenido",
+    input_data_before_enhancing_property: "Necesitas ingresar datos en otras propiedades antes de mejorar esta.",
+    based_on_rest_of_entity: "Basado en el resto de la entidad",
+    this_field: "este campo",
+    subscription_needed: "Se necesita una suscripción válida para usar esta función.",
+    login_to_enhance: "Necesitas iniciar sesión para mejorar datos",
+    no_fields_updated: "No se actualizaron campos",
+    ask_ai_to_write: "Pídele a la IA que escriba",
+    instructions: "Instrucciones"
+};

package/src/locales/fr.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsFr = {
+    autofill: "Remplissage automatique",
+    autofill_current_content: "Remplissage automatique basé sur le contenu actuel",
+    set_id_first: "Vous devez d'abord définir un identifiant",
+    provide_instructions: "Fournir des instructions pour l'amélioration des données",
+    fill_missing_fields: "Remplir les champs manquants",
+    translate_missing_content: "Traduire le contenu manquant",
+    add_2_paragraphs_to: "Ajouter 2 paragraphes au contenu",
+    use_openai_generate_content: "Utilisez Google Vertex AI ou OpenAI pour générer du contenu automatiquement.",
+    autofill_this_field: "Remplir automatiquement ce champ",
+    autofill_property: "Propriété de remplissage automatique",
+    input_data_before_enhancing: "Saisir des données avant l'amélioration",
+    input_data_before_enhancing_property: "Vous devez saisir des données dans d'autres propriétés avant d'améliorer celle-ci.",
+    based_on_rest_of_entity: "Basé sur le reste de l'entité",
+    this_field: "ce champ",
+    subscription_needed: "Un abonnement valide est nécessaire pour utiliser cette fonctionnalité.",
+    login_to_enhance: "Vous devez être connecté pour améliorer les données",
+    no_fields_updated: "Aucun champ n'a été mis à jour",
+    ask_ai_to_write: "Demander à l'IA d'écrire",
+    instructions: "Instructions"
+};

package/src/locales/hi.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsHi = {
+    autofill: "ऑटोफ़िल",
+    autofill_current_content: "वर्तमान सामग्री के आधार पर ऑटोफ़िल करें",
+    set_id_first: "पहले एक ID सेट करना आवश्यक है",
+    provide_instructions: "डेटा एन्हांसमेंट के लिए निर्देश प्रदान करें",
+    fill_missing_fields: "अनुपस्थित फ़ील्ड भरें",
+    translate_missing_content: "अनुपस्थित सामग्री अनुवाद करें",
+    add_2_paragraphs_to: "सामग्री में 2 पैराग्राफ जोड़ें",
+    use_openai_generate_content: "सामग्री स्वचालित रूप से जनरेट करने के लिए Google Vertex AI या OpenAI का उपयोग करें।",
+    autofill_this_field: "इस फ़ील्ड को ऑटोफ़िल करें",
+    autofill_property: "ऑटोफ़िल प्रॉपर्टी",
+    input_data_before_enhancing: "एन्हांस करने से पहले कुछ डेटा दर्ज करें",
+    input_data_before_enhancing_property: "इस प्रॉपर्टी को एन्हांस करने से पहले आपको अन्य प्रॉपर्टीज़ में डेटा दर्ज करना होगा।",
+    based_on_rest_of_entity: "एंटिटी के बाकी भाग के आधार पर",
+    this_field: "यह फ़ील्ड",
+    subscription_needed: "इस फ़ंक्शन का उपयोग करने के लिए एक वैध सब्सक्रिप्शन आवश्यक है।",
+    login_to_enhance: "डेटा एन्हांस करने के लिए लॉग इन करना आवश्यक है",
+    no_fields_updated: "कोई फ़ील्ड अपडेट नहीं हुई",
+    ask_ai_to_write: "AI से लिखने के लिए कहें",
+    instructions: "निर्देश"
+};

package/src/locales/it.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsIt = {
+    autofill: "Compilazione automatica",
+    autofill_current_content: "Compilazione automatica basata sul contenuto attuale",
+    set_id_first: "È necessario impostare prima un ID",
+    provide_instructions: "Fornisci istruzioni per il miglioramento dei dati",
+    fill_missing_fields: "Compila i campi mancanti",
+    translate_missing_content: "Traduci il contenuto mancante",
+    add_2_paragraphs_to: "Aggiungi 2 paragrafi al contenuto",
+    use_openai_generate_content: "Usa Google Vertex AI o OpenAI per generare contenuti automaticamente.",
+    autofill_this_field: "Compila automaticamente questo campo",
+    autofill_property: "Proprietà di compilazione automatica",
+    input_data_before_enhancing: "Inserisci alcuni dati prima di migliorare",
+    input_data_before_enhancing_property: "Devi inserire dati in altre proprietà prima di migliorare questa.",
+    based_on_rest_of_entity: "Basato sul resto dell'entità",
+    this_field: "questo campo",
+    subscription_needed: "È necessario un abbonamento valido per utilizzare questa funzione.",
+    login_to_enhance: "Devi essere connesso per migliorare i dati",
+    no_fields_updated: "Nessun campo è stato aggiornato",
+    ask_ai_to_write: "Chiedi all'IA di scrivere",
+    instructions: "Istruzioni"
+};

package/src/locales/pt.ts

@@ -0,0 +1,21 @@
+export const dataEnhancementTranslationsPt = {
+    autofill: "Preenchimento automático",
+    autofill_current_content: "Preencher automaticamente com base no conteúdo atual",
+    set_id_first: "Deve definir primeiro um ID",
+    provide_instructions: "Forneça instruções ao Data Enhancement",
+    fill_missing_fields: "Preencher campos em falta",
+    translate_missing_content: "Traduzir conteúdo em falta",
+    add_2_paragraphs_to: "Adicionar 2 parágrafos ao conteúdo",
+    use_openai_generate_content: "Use o Google Vertex AI ou OpenAI para gerar conteúdo automaticamente.",
+    autofill_this_field: "Preencher automaticamente este campo",
+    autofill_property: "Preencher propriedade automaticamente",
+    input_data_before_enhancing: "Introduza alguns dados antes de melhorar",
+    input_data_before_enhancing_property: "Precisa de introduzir alguns dados noutras propriedades antes de melhorar esta.",
+    based_on_rest_of_entity: "Com base no resto da entidade",
+    this_field: "este campo",
+    subscription_needed: "É necessária uma subscrição válida para usar esta função.",
+    login_to_enhance: "Precisa de iniciar sessão para melhorar os dados",
+    no_fields_updated: "Nenhum campo foi atualizado",
+    ask_ai_to_write: "Pedir à IA para escrever",
+    instructions: "Instruções"
+};

package/src/tests/diffStrings.test.ts

@@ -0,0 +1,128 @@
+// Import the diffStrings function from your module
+// import { diffStrings } from './your-module';
+
+import { Change, diffStrings } from "../utils/diffStrings";
+
+describe("diffStrings", () => {
+    test("equal strings", () => {
+        const oldStr = "This is a test string";
+        const newStr = "This is a test string";
+        const expected: Change[] = [
+            {
+                type: "equal",
+                value: "This is a test string"
+            }
+        ];
+        expect(diffStrings(oldStr, newStr)).toEqual(expected);
+    });
+
+    test("insertions only", () => {
+        const oldStr = "This is a test string";
+        const newStr = "This is a new test string";
+        const expected: Change[] = [
+            {
+                type: "equal",
+                value: "This is a"
+            },
+            {
+                type: "insert",
+                value: " new"
+            },
+            {
+                type: "equal",
+                value: " test string"
+            }
+        ];
+        expect(diffStrings(oldStr, newStr)).toEqual(expected);
+    });
+
+    test("deletions only", () => {
+        const oldStr = "This is an old test string";
+        const newStr = "This is a test string";
+        const expected: Change[] = [
+            {
+                type: "equal",
+                value: "This is a"
+            },
+            {
+                type: "delete",
+                value: "n old"
+            },
+            {
+                type: "equal",
+                value: " test string"
+            }
+        ];
+        expect(diffStrings(oldStr, newStr)).toEqual(expected);
+    });
+
+    test("insertions and deletions", () => {
+        const oldStr = "This is an old test string";
+        const newStr = "This is a new modified test string";
+        // The LCS algorithm finds the longest common substrings correctly
+        // Even though the output is more granular, it correctly represents the diff
+        const expected: Change[] = [
+            {
+                type: "equal",
+                value: "This is a"
+            },
+            {
+                type: "insert",
+                value: " "
+            },
+            {
+                type: "equal",
+                value: "n"
+            },
+            {
+                type: "insert",
+                value: "ew"
+            },
+            {
+                type: "equal",
+                value: " "
+            },
+            {
+                type: "insert",
+                value: "m"
+            },
+            {
+                type: "equal",
+                value: "o"
+            },
+            {
+                type: "delete",
+                value: "l"
+            },
+            {
+                type: "insert",
+                value: "difie"
+            },
+            {
+                type: "equal",
+                value: "d test string"
+            }
+        ];
+        expect(diffStrings(oldStr, newStr)).toEqual(expected);
+    });
+
+    test("completely different strings", () => {
+        const oldStr = "Old string";
+        const newStr = "New string";
+        const expected: Change[] = [
+            {
+                type: "delete",
+                value: "Old"
+            },
+            {
+                type: "insert",
+                value: "New"
+            },
+            {
+                type: "equal",
+                value: " string"
+            }
+        ];
+        expect(diffStrings(oldStr, newStr)).toEqual(expected);
+    });
+});