@rebasepro/plugin-data-enhancement 0.0.1-canary.000dc36

package/package.json

@@ -0,0 +1,86 @@
+{
+  "name": "@rebasepro/plugin-data-enhancement",
+  "type": "module",
+  "version": "0.0.1-canary.000dc36",
+  "main": "./dist/index.umd.js",
+  "module": "./dist/index.es.js",
+  "types": "./dist/index.d.ts",
+  "source": "src/index.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "development": "./src/index.ts",
+      "import": "./dist/index.es.js",
+      "require": "./dist/index.umd.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "dependencies": {
+    "react-compiler-runtime": "1.0.0",
+    "@rebasepro/admin": "0.0.1-canary.000dc36",
+    "@rebasepro/core": "0.0.1-canary.000dc36",
+    "@rebasepro/ui": "0.0.1-canary.000dc36",
+    "@rebasepro/types": "0.0.1-canary.000dc36",
+    "@rebasepro/utils": "0.0.1-canary.000dc36",
+    "@rebasepro/common": "0.0.1-canary.000dc36"
+  },
+  "peerDependencies": {
+    "react": ">=19.0.0",
+    "react-dom": ">=19.0.0",
+    "react-router": ">=6.28.0",
+    "react-router-dom": ">=6.28.0"
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "devDependencies": {
+    "@testing-library/jest-dom": "^6.9.1",
+    "@types/jest": "^29.5.14",
+    "@types/react": "^19.0.8",
+    "@types/react-dom": "^19.0.3",
+    "@vitejs/plugin-react": "^4.3.4",
+    "babel-jest": "^29.7.0",
+    "babel-plugin-react-compiler": "^19.0.0-beta-af1b7da-20250417",
+    "eslint-plugin-react-compiler": "^19.1.0-rc.2",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.4.5",
+    "typescript": "^5.9.3",
+    "vite": "^7.2.4"
+  },
+  "jest": {
+    "transform": {
+      "^.+\\.tsx?$": "ts-jest"
+    },
+    "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$",
+    "moduleFileExtensions": [
+      "ts",
+      "tsx",
+      "js",
+      "jsx",
+      "json",
+      "node"
+    ]
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "gitHead": "d935eefa5aa8d1009a2398cfac2c1e4ee9aeb6b6",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build && tsc --emitDeclarationOnly -p tsconfig.prod.json",
+    "test": "jest --passWithNoTests"
+  }
+}

package/src/api.ts

@@ -0,0 +1,198 @@
+import {
+    DataEnhancementRequest,
+    EnhancedDataResult,
+    InputEntity,
+    InputProperty,
+    SamplePromptsResult
+} from "./types/data_enhancement_controller";
+import { EntityValues } from "@rebasepro/types";
+import { flatMapEntityValues } from "./utils/values";
+
+// const DEFAULT_SERVER = "http://localhost:5001/rebase-dev-2da42/europe-west3/api"; // Local
+
+const DEFAULT_SERVER = "https://api.rebase.pro";
+
+export async function enhanceDataAPIStream<M extends Record<string, unknown>>(props: {
+    apiKey: string,
+    entityId?: string | number,
+    entityName: string,
+    entityDescription?: string,
+    propertyKey?: string,
+    propertyInstructions?: string;
+    values: EntityValues<M>,
+    path: string,
+    properties: Record<string, InputProperty>,
+
+    instructions?: string,
+    firebaseToken: string,
+    onUpdate: (suggestions: Record<string, any>) => void;
+    onUpdateDelta: (propertyKey: string, partialValue: any) => void;
+    onError: (error: Error) => void;
+    onEnd: (result: EnhancedDataResult) => void;
+    host?: string;
+}) {
+
+    const flatValues = flatMapEntityValues(props.values);
+
+    const properties = props.properties;
+
+    const inputEntity: InputEntity = {
+        entityId: props.entityId,
+        values: flatValues
+    }
+
+    const request: DataEnhancementRequest = {
+        inputEntity,
+        properties,
+        entityName: props.entityName,
+        entityDescription: props.entityDescription,
+        propertyKey: props.propertyKey,
+        propertyInstructions: props.propertyInstructions,
+        instructions: props.instructions
+    };
+
+    console.debug("enhanceDataAPIStream", request);
+
+    return fetch((props.host ?? DEFAULT_SERVER) + "/data/enhance_stream/",
+        {
+            // mode: "no-cors",
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Basic ${props.firebaseToken}`,
+                "x-de-api-key": `Basic ${props.apiKey}`
+                // "x-de-version": version
+            },
+            body: JSON.stringify(request)
+        })
+        .then(async (res) => {
+            if (!res.ok) {
+                console.error("enhanceDataAPIStream error", res)
+                throw await res.json();
+            }
+            const reader = res.body?.getReader();
+            if (!reader) {
+                throw new Error("No reader");
+            }
+
+            for await (const chunk of readChunks(reader)) {
+                const str = new TextDecoder().decode(chunk);
+                try {
+                    str.split("&$# ").forEach((s) => {
+                        if (s && s.length > 0) {
+                            const data = JSON.parse(s.trim());
+                            if (data.type === "suggestion_delta")
+                                props.onUpdateDelta(data.data.propertyKey, data.data.partialValue);
+                            else if (data.type === "suggestion")
+                                props.onUpdate(data.data);
+                            else if (data.type === "result")
+                                props.onEnd(data.data);
+                        }
+                    });
+                } catch (e: any) {
+                    console.error("str", str);
+                    console.error("Error parsing stream", e);
+                    props.onError(e);
+                }
+            }
+
+        });
+
+}
+
+function readChunks(reader: ReadableStreamDefaultReader) {
+    return {
+        async *[Symbol.asyncIterator]() {
+            let readResult = await reader.read();
+            while (!readResult.done) {
+                yield readResult.value;
+                readResult = await reader.read();
+            }
+        }
+    };
+}
+
+export async function fetchEntityPromptSuggestion<M extends object>(props: {
+    input?: string,
+    entityName: string,
+    firebaseToken: string,
+    apiKey: string,
+    host?: string
+}): Promise<SamplePromptsResult> {
+
+    return fetch((props.host ?? DEFAULT_SERVER) + "/data/prompt_autocomplete/",
+        {
+            // mode: "no-cors",
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Basic ${props.firebaseToken}`,
+                "x-de-api-key": `Basic ${props.apiKey}`
+            },
+            body: JSON.stringify({
+                entityName: props.entityName,
+                input: props.input ?? null
+            })
+        })
+        .then(async (res) => {
+            const data = await res.json();
+            if (!res.ok) {
+                console.error("fetchEntityPromptSuggestion", data);
+                throw Error(data.message);
+            }
+            return {
+                prompts: data.data.prompts.map((e: string) => ({
+                    prompt: e,
+                    type: "sample"
+                }))
+            };
+        });
+
+}
+
+export async function autocompleteStream(props: {
+    firebaseToken: string,
+    textBefore?: string,
+    textAfter: string,
+    host?: string;
+    onUpdate: (delta: string) => void;
+}) {
+
+    let result = "";
+    return fetch((props.host ?? DEFAULT_SERVER) + "/data/autocomplete/",
+        {
+            // mode: "no-cors",
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Basic ${props.firebaseToken}`
+                // "x-de-version": version
+            },
+            body: JSON.stringify({
+                textBefore: props.textBefore,
+                textAfter: props.textAfter
+            })
+        })
+        .then(async (res) => {
+            if (!res.ok) {
+                console.error("enhanceDataAPIStream error", res)
+                throw await res.json();
+            }
+            const reader = res.body?.getReader();
+            if (!reader) {
+                throw new Error("No reader");
+            }
+
+            for await (const chunk of readChunks(reader)) {
+                const str = new TextDecoder().decode(chunk);
+                result += str;
+                console.debug("Autocomplete update:", str);
+                props.onUpdate(str);
+            }
+
+        }).then(() => {
+            console.debug("Autocomplete result:", result);
+            return result;
+        });
+
+}

package/src/components/DataEnhancementControllerProvider.tsx

@@ -0,0 +1,339 @@
+import React, { PropsWithChildren, useCallback, useContext, useEffect, useMemo, useRef, useState } from "react";
+
+import {
+    DataEnhancementController,
+    EnhancedDataResult,
+    EnhanceParams,
+    InputProperty
+} from "../types/data_enhancement_controller";
+import {
+    useAuthController,
+    useCustomizationController,
+    useSnackbarController
+} from "@rebasepro/core";
+import { useUrlController } from "@rebasepro/admin";
+import { DataDriver, Entity, EntityCollection, PluginFormActionProps } from "@rebasepro/types";
+import { enhanceDataAPIStream, fetchEntityPromptSuggestion } from "../api";
+import { getAppendableSuggestion } from "../utils/suggestions";
+import { getSimplifiedProperties } from "../utils/properties";
+import { useEditorAIController } from "../editor/useEditorAIController";
+import { getValueInPath } from "@rebasepro/utils";
+
+export const DataEnhancementControllerContext = React.createContext<DataEnhancementController>(null! as DataEnhancementController);
+
+export type DataEnhancementControllerProviderProps = {
+
+    apiKey: string;
+
+    getConfigForPath?: (props: {
+        path: string,
+        collection: EntityCollection
+    }) => boolean;
+
+    host?: string;
+}
+
+export const useDataEnhancementController = (): DataEnhancementController => useContext(DataEnhancementControllerContext);
+
+function getPropertyFromKey(properties: Record<string, InputProperty>, propertyKey: string) {
+    if (propertyKey in properties) {
+        return properties[propertyKey];
+    } else {
+        //split the property key
+        const split = propertyKey.split(".");
+        if (split.length === 1) {
+            return undefined;
+        }
+        const parentKey = split.slice(0, split.length - 1).join(".");
+        return getPropertyFromKey(properties, parentKey);
+
+    }
+}
+
+export function DataEnhancementControllerProvider({
+    apiKey,
+    getConfigForPath,
+    children,
+    host,
+    path,
+    collection,
+    formContext
+}: PropsWithChildren<DataEnhancementControllerProviderProps & PluginFormActionProps<any>>) {
+
+    const [enabled, setEnabled] = useState(false);
+    const [suggestions, setSuggestions] = useState<Record<string, string | number>>({});
+    const [loadingSuggestions, setLoadingSuggestions] = useState<string[]>([]);
+
+    const enhancingInProgress = useRef(false);
+
+    const authController = useAuthController();
+    const snackbarController = useSnackbarController();
+
+
+    const properties = useMemo(() => getSimplifiedProperties(collection.properties, formContext?.values ?? {}), [formContext?.values]);
+    // const preEnhanceValuesRef = React.useRef(formContext?.values ?? {});
+    const valuesRef = React.useRef(formContext?.values ?? {});
+    useEffect(() => {
+        if (!enhancingInProgress.current)
+            valuesRef.current = formContext?.values ?? {};
+    }, [formContext?.values]);
+
+    const allowReferenceDataSelection = false;
+
+    const updateConfig = useCallback(async () => {
+        if (!getConfigForPath) return;
+        const config = getConfigForPath({
+            path,
+            collection
+        });
+        if (config) {
+            setEnabled(true);
+        }
+    }, [collection, getConfigForPath, path]);
+
+    useEffect(() => {
+        if (!getConfigForPath) {
+            setEnabled(true);
+        } else {
+            updateConfig();
+        }
+
+    }, [getConfigForPath, updateConfig]);
+
+
+    const urlController = useUrlController();
+
+    const clearSuggestion = useCallback((propertyKey: string) => {
+        setSuggestions((prev) => {
+            //remove propertyKey from prev
+            const {
+                [propertyKey]: _,
+                ...rest
+            } = prev;
+            return rest;
+        });
+    }, []);
+
+    const appendValueDelta = useCallback((propertyKey: string, delta: string) => {
+
+        const property = getPropertyFromKey(properties, propertyKey);
+        if (delta === null || property?.disabled) {
+            return;
+        }
+
+        // clearSuggestion(propertyKey);
+        const value = getValueInPath(valuesRef.current, propertyKey);
+
+        const currentValue = value ? (value as string) + "" : "";
+        const updatedValue = currentValue + delta;
+        // if (currentValue.length === 0) updatedValue = updatedValue.trimStart();
+        valuesRef.current = {
+            ...valuesRef.current,
+            [propertyKey]: updatedValue
+        };
+        formContext?.setFieldValue(propertyKey, updatedValue, false);
+        setSuggestions(prev => ({
+            ...prev,
+            [propertyKey]: (prev[propertyKey] ?? "") + delta
+        }));
+    }, [properties, formContext]);
+
+    const updateSuggestedValues = useCallback((currentValues: object, updatedValues: Record<string, string | number>, replaceValues: boolean) => {
+
+        setLoadingSuggestions((prev) => {
+            return prev.filter(p => !Object.keys(updatedValues).includes(p));
+        });
+
+        Object.entries(updatedValues).forEach(([propertyKey, suggestion]) => {
+
+            const value = getValueInPath(currentValues, propertyKey);
+            const property = getPropertyFromKey(properties, propertyKey);
+
+            if (!property || suggestion === null || property?.disabled) {
+                return;
+            }
+
+            if (typeof suggestion === "number") {
+                formContext?.setFieldValue(propertyKey, suggestion);
+                return;
+            }
+
+            if (replaceValues) {
+                formContext?.setFieldValue(propertyKey, suggestion);
+                return;
+            }
+
+            const appendableValue = getAppendableSuggestion(suggestion, value);
+
+            const currentValue = value ? (value as string) + "" : "";
+            if (appendableValue) {
+                formContext?.setFieldValue(propertyKey, suggestion);
+            } else {
+                const multiline = property?.fieldConfigId === "multiline" || property?.fieldConfigId === "markdown";
+                const trimmedValue = currentValue.trimEnd();
+                if (multiline && (trimmedValue.endsWith(".") || trimmedValue.endsWith("?") || trimmedValue.endsWith("!") || trimmedValue.endsWith(":"))) {
+                    formContext?.setFieldValue(propertyKey, trimmedValue + "\n\n" + (suggestion as string).trimStart());
+                } else {
+                    formContext?.setFieldValue(propertyKey, trimmedValue + (trimmedValue.length > 0 ? " " : "") + (suggestion as string));
+                }
+            }
+        });
+
+        setSuggestions(prev => ({
+            ...prev,
+            ...Object.keys(updatedValues)
+                .reduce((acc, key) => {
+                    const value = getValueInPath(formContext?.values, key);
+                    const suggestion = updatedValues[key];
+                    return {
+                        ...acc,
+                        [key]: getAppendableSuggestion(suggestion, value) ?? suggestion
+                    };
+                }, {})
+        }));
+    }, [properties, formContext]);
+
+    const displayNeededSubscriptionSnackbar = useCallback((projectId: any) => {
+        snackbarController.open({
+            type: "warning",
+            message: "A valid subscription is needed in order to use this function.",
+            autoHideDuration: 4000
+        });
+    }, [snackbarController]);
+
+    const editorAIController = useEditorAIController({ getAuthToken: authController.getAuthToken });
+
+    const clearAllSuggestions = useCallback(() => {
+        setSuggestions({});
+    }, []);
+
+    const enhance = useCallback(async (props: EnhanceParams<any>): Promise<EnhancedDataResult | null> => {
+
+        if (!authController.user) {
+            snackbarController.open({
+                type: "warning",
+                message: "You need to be logged in to enhance data"
+            });
+            return Promise.reject(new Error("Not logged in"));
+        }
+
+        const resolvedPath = urlController.resolveDatabasePathsFrom(path);
+        const firebaseToken = await authController.getAuthToken();
+
+        if (props.propertyKey) {
+            clearSuggestion(props.propertyKey)
+        } else {
+            clearAllSuggestions();
+        }
+
+        setLoadingSuggestions((prev) => [...prev, ...(props.propertyKey ? [props.propertyKey] : Object.keys(properties))]);
+        enhancingInProgress.current = true;
+
+        const currentValues = valuesRef.current ?? {};
+
+        return new Promise((resolve, reject) => {
+            function onError(e: any) {
+                setLoadingSuggestions([]);
+                if (e.code === "payment-required") {
+                    const projectId = e.data.projectId;
+                    displayNeededSubscriptionSnackbar(projectId);
+                } else {
+                    console.error("Enhance error", e);
+                }
+                reject(e);
+                enhancingInProgress.current = false;
+            }
+
+            try {
+                enhanceDataAPIStream({
+                    ...props,
+                    host,
+                    apiKey,
+                    properties,
+                    path: resolvedPath,
+                    entityName: collection.singularName ?? collection.name,
+                    entityDescription: collection.description,
+
+                    firebaseToken,
+                    onUpdate: (suggestions) => {
+                        console.debug("de onUpdate", suggestions);
+                        updateSuggestedValues(currentValues, suggestions, props.replaceValues ?? false);
+                    },
+                    onUpdateDelta: (propertyKey: string, partialValue: string) => {
+                        // console.debug("de delta", propertyKey, partialValue);
+                        appendValueDelta(propertyKey, partialValue);
+                    },
+                    onError,
+                    onEnd: (result) => {
+                        console.debug("de onEnd", result);
+                        if (result.errors) {
+                            result.errors.forEach((error) => {
+                                snackbarController.open({
+                                    type: "warning",
+                                    message: error
+                                })
+                            });
+                        }
+                        if (Object.keys(result.suggestions).length === 0) {
+                            snackbarController.open({
+                                type: "info",
+                                autoHideDuration: 1800,
+                                message: "No fields were updated"
+                            })
+                        }
+                        setLoadingSuggestions([]);
+                        resolve(result);
+                        enhancingInProgress.current = false;
+                    }
+                }).catch(onError);
+            } catch (e: any) {
+                onError(e);
+            }
+        });
+    }, [
+        authController, urlController, path, clearSuggestion, clearAllSuggestions,
+        properties, host, apiKey, collection, updateSuggestedValues, appendValueDelta, displayNeededSubscriptionSnackbar, snackbarController
+    ]);
+
+    const getSamplePrompts = useCallback(async (entityName: string, input?: string) => {
+        const firebaseToken = await authController.getAuthToken()
+        return fetchEntityPromptSuggestion({
+            host,
+            entityName,
+            firebaseToken,
+            apiKey,
+            input
+        });
+    }, [apiKey, authController.getAuthToken, host]);
+
+    const dataEnhancementController: DataEnhancementController = useMemo(() => ({
+        enabled,
+        suggestions,
+        clearSuggestion,
+        enhance,
+        allowReferenceDataSelection,
+        clearAllSuggestions,
+        getSamplePrompts,
+        loadingSuggestions,
+        editorAIController
+    }), [
+        enabled,
+        suggestions,
+        clearSuggestion,
+        enhance,
+        allowReferenceDataSelection,
+        clearAllSuggestions,
+        getSamplePrompts,
+        loadingSuggestions,
+        editorAIController
+    ]);
+
+    return (
+        <DataEnhancementControllerContext.Provider
+            value={dataEnhancementController}>
+            {children}
+        </DataEnhancementControllerContext.Provider>
+    );
+}
+
+