@rebasepro/plugin-data-enhancement 0.0.1-canary.000dc36

package/src/types/data_enhancement_controller.tsx

@@ -0,0 +1,75 @@
+import { EntityValues } from "@rebasepro/types";
+import { EditorAIController } from "@rebasepro/admin";
+
+export type EnhanceParams<M extends Record<string, unknown>> = {
+    entityId?: string | number;
+    propertyKey?: string;
+    propertyInstructions?: string;
+    values: EntityValues<M>;
+    instructions?: string;
+    replaceValues: boolean;
+};
+
+export type DataEnhancementController = {
+    /**
+     * Whether the data enhancement is enabled for the current path
+     */
+    enabled: boolean;
+    suggestions: Record<string, string | number>;
+    enhance: <M extends Record<string, unknown>>(props: EnhanceParams<M>) => Promise<EnhancedDataResult | null>;
+    clearSuggestion: (key: string, suggestion: string | number) => void;
+    allowReferenceDataSelection: boolean;
+    clearAllSuggestions: () => void;
+    getSamplePrompts: (entityName: string, input?: string) => Promise<SamplePromptsResult>;
+    loadingSuggestions: string[],
+    editorAIController?: EditorAIController;
+}
+
+export type EnhancedDataResult = {
+    entityId?: string | number;
+    suggestions: {
+        [key: string]: string[];
+    };
+    errors: string[];
+    usage: { promptTokens?: number, completionTokens?: number, totalTokens?: number }
+}
+
+export type SamplePrompt = {
+    prompt: string;
+    type: "recent" | "sample";
+}
+
+export type SamplePromptsResult = {
+    prompts: SamplePrompt[];
+    host?: string;
+};
+
+export type DataEnhancementRequest = {
+    entityName: string;
+    entityDescription?: string;
+    inputEntity: InputEntity;
+    properties: Record<string, InputProperty>;
+    propertyKey?: string;
+    propertyInstructions?: string,
+    instructions?: string;
+};
+
+export type InputEntity = {
+    entityId?: string | number;
+    values: Record<string, any>;
+};
+
+export type InputProperty = {
+    name?: string;
+    description?: string;
+    type: string;
+    fieldConfigId: string;
+    enum?: string[];
+    disabled?: boolean;
+    of?: InputProperty;
+    oneOf?: {
+        properties: Record<string, InputProperty>;
+        typeField?: string;
+        valueField?: string;
+    };
+}

package/src/types/subscriptions_message_props.tsx

@@ -0,0 +1 @@
+export type SubscriptionMessageProps = { projectId: string };

package/src/useDataEnhancementPlugin.tsx

@@ -0,0 +1,66 @@
+import React from "react";
+
+import { EntityCollection, RebasePlugin, User } from "@rebasepro/types";
+import { DataEnhancementControllerProvider } from "./components/DataEnhancementControllerProvider";
+import { FormEnhanceAction } from "./components/FormEnhanceAction";
+
+const DEFAULT_API_KEY = "fcms-U9jdDii0xXWSDC34asfrf54lbkFJBfKfRWcEDEwdc4V5wDWEDF";
+
+export interface DataEnhancementPluginProps {
+
+    apiKey?: string;
+
+    /**
+     * Use this function to determine if the data enhancement plugin should be enabled for a given path.
+     * If this function is not provided, the plugin will be enabled for all paths.
+     * If the function returns false, the plugin will be disabled for the given path.
+     * You can also return a configuration object to override the default configuration.
+     *
+     * @param path
+     * @param collection
+     */
+    getConfigForPath?: (props: {
+        path: string,
+        collection: EntityCollection,
+        user: User | null
+    }) => boolean;
+
+    /**
+     * Host to use for the data enhancement API.
+     * This prop is only use in development mode.
+     */
+    host?: string;
+}
+
+/**
+ * Use this hook to initialise the data enhancement plugin.
+ * This is likely the only hook you will need to use.
+ * @param props
+ */
+export function useDataEnhancementPlugin(props?: DataEnhancementPluginProps): RebasePlugin {
+
+    const apiKey = props?.apiKey ?? DEFAULT_API_KEY;
+    const getConfigForPath = props?.getConfigForPath;
+
+    return React.useMemo(() => ({
+        key: "data_enhancement",
+        slots: [
+            {
+                slot: "form.actions",
+                Component: FormEnhanceAction,
+                order: 40
+            }
+        ],
+        providers: [
+            {
+                scope: "form" as const,
+                Component: DataEnhancementControllerProvider as React.ComponentType<any>,
+                props: {
+                    apiKey,
+                    getConfigForPath,
+                    host: props?.host
+                }
+            }
+        ]
+    }), [apiKey, getConfigForPath, props?.host]);
+}

package/src/utils/diffStrings.ts

@@ -0,0 +1,70 @@
+type ChangeType = "equal" | "delete" | "insert";
+
+export interface Change {
+    type: ChangeType;
+    value: string;
+}
+
+export function diffStrings(oldStr: string, newStr: string): Change[] {
+    // Find the longest common substring
+    function longestCommonSubstring(s1: string, s2: string): string {
+        const longest = { start: 0,
+length: 0 };
+        const matrix = new Array(s1.length + 1).fill(null).map(() => new Array(s2.length + 1).fill(0));
+
+        for (let i = 1; i <= s1.length; i++) {
+            for (let j = 1; j <= s2.length; j++) {
+                if (s1[i - 1] === s2[j - 1]) {
+                    matrix[i][j] = matrix[i - 1][j - 1] + 1;
+                    if (matrix[i][j] > longest.length) {
+                        longest.start = i - matrix[i][j];
+                        longest.length = matrix[i][j];
+                    }
+                }
+            }
+        }
+
+        return s1.slice(longest.start, longest.start + longest.length);
+    }
+
+    // Recursively find changes and create Change objects
+    function findChanges(s1: string, s2: string): Change[] {
+        if (s1 === s2) return s1.length > 0 ? [{
+            type: "equal",
+            value: s1
+        }] : [];
+
+        const common = longestCommonSubstring(s1, s2);
+
+        if (common.length === 0) {
+            const changes: Change[] = [];
+            if (s1.length > 0) {
+                changes.push({ type: "delete",
+value: s1 });
+            }
+            if (s2.length > 0) {
+                changes.push({ type: "insert",
+value: s2 });
+            }
+            return changes;
+        }
+
+        const s1Before = s1.slice(0, s1.indexOf(common));
+        const s1After = s1.slice(s1.indexOf(common) + common.length);
+        const s2Before = s2.slice(0, s2.indexOf(common));
+        const s2After = s2.slice(s2.indexOf(common) + common.length);
+
+        const changesBefore = findChanges(s1Before, s2Before);
+        const changesAfter = findChanges(s1After, s2After);
+
+        return [
+            ...changesBefore,
+            { type: "equal",
+value: common },
+            ...changesAfter
+        ];
+    }
+
+    return findChanges(oldStr, newStr);
+}
+

package/src/utils/properties.ts

@@ -0,0 +1,168 @@
+import { getFieldId } from "@rebasepro/admin";
+import { EnumValues, Properties, Property } from "@rebasepro/types";
+import { isPropertyBuilder } from "@rebasepro/common";
+import { InputProperty } from "../types/data_enhancement_controller";
+import { getValueInPath } from "@rebasepro/utils";
+
+export function getSimplifiedProperties<M extends Record<string, any>>(properties: Properties, values: M, path = ""): Record<string, InputProperty> {
+    if (!properties) return {};
+    return Object.entries(properties)
+        .map(([key, property]) => {
+            if (isPropertyBuilder(property)) return {};
+            const fullKey = path ? `${path}.${key}` : key;
+            const valueInPath = getValueInPath(values, fullKey);
+            return getSimplifiedProperty(property, fullKey, valueInPath)
+        })
+        .reduce((a, b) => ({ ...a,
+...b }), {});
+}
+
+function getSimpleProperty(property: Property): InputProperty {
+    const fieldId = getFieldId(property);
+    if (!fieldId) {
+        console.error("No fieldId found for property", property);
+        throw new Error("Field id not found");
+    }
+    return {
+        name: property.name,
+        description: property.description,
+        type: property.type,
+        fieldConfigId: fieldId,
+        enum: "enum" in property && property.enum
+            ? getSimpleEnumValues(property.enum)
+            : undefined,
+        disabled: Boolean(property.ui?.disabled || property.ui?.readOnly)
+    };
+}
+
+function getSimplifiedProperty(property: Property, path: string, value?: any): Record<string, InputProperty> {
+    if (isPropertyBuilder(property)) return {};
+    if (property.type === "array") {
+
+        if (property.of && !Array.isArray(property.of) && !isPropertyBuilder(property.of)) {
+            const arrayParentProperty: InputProperty = {
+                name: property.name,
+                description: property.description,
+                type: property.type,
+                fieldConfigId: "repeat",
+                disabled: Boolean(property.ui?.disabled || property.ui?.readOnly),
+                of: getSimpleProperty(property.of as Property)
+            };
+
+            const result = { [path]: arrayParentProperty };
+            // if (Array.isArray(value)) {
+            //     result = {
+            //         ...result,
+            //         ...value
+            //             .map((v, i) => getSimplifiedProperty(property.of, `${path}.${i}`, v))
+            //             .reduce((a, b) => ({ ...a, ...b }), {})
+            //     };
+            // }
+            //
+            // const existingValuesCount = Array.isArray(value) ? value.length : 0;
+            //
+            // const newValuesCount = property.of && !isPropertyBuilder<any, any>(property.of) && (property.of as Property).type === "map" ? 1 : 3;
+            // result = {
+            //     ...result,
+            //     // ...Array.from(Array(newValuesCount))
+            //     //     .map((v, i) => getSimplifiedProperty(property.of, `${path}.${i + existingValuesCount}`, v))
+            //     //     .reduce((a, b) => ({ ...a, ...b }), {})
+            // }
+
+            return result;
+        } else if (property.oneOf) {
+
+            const arrayParentProperty: InputProperty = {
+                name: property.name,
+                description: property.description,
+                type: property.type,
+                fieldConfigId: "block",
+                disabled: Boolean(property.ui?.disabled || property.ui?.readOnly),
+                oneOf: {
+                    typeField: property.oneOf.typeField,
+                    valueField: property.oneOf.valueField,
+                    properties: Object.entries(property.oneOf.properties)
+                        .map(([key, prop]) => ({ [key]: getSimpleProperty(prop) }))
+                        .reduce((a, b) => ({ ...a,
+...b }), {})
+                }
+            };
+
+            if (!Array.isArray(value)) {
+                return { [path]: arrayParentProperty };
+            }
+
+            return value.map((v, i) => {
+                if (v == null) return {};
+                const typeKey = property.oneOf!.typeField ?? "type";
+                const oneOfType = v[typeKey];
+                const valueKey = property.oneOf!.valueField ?? "value";
+                const oneOfValue = v[valueKey];
+                const childProperty = property.oneOf!.properties[oneOfType];
+                if (childProperty === undefined) {
+                    console.error(`No property found for type ${oneOfType}`, property.oneOf!.properties);
+                    return {};
+                }
+                const simplifiedProperty = getSimplifiedProperty(childProperty, `${path}.${i}.${valueKey}`, oneOfValue);
+                return {
+                    [`${path}.${i}.${typeKey}`]: oneOfType,
+                    ...simplifiedProperty
+                };
+            }).reduce((a, b) => ({ ...a,
+...b }), { [path]: arrayParentProperty });
+        }
+    } else if (property.type === "map") {
+        if (property.properties) {
+            const mapProperties: Record<string, InputProperty> = Object.entries(property.properties)
+                .map(([key, childProperty]) => {
+                    const childValue = value?.[key];
+                    return getSimplifiedProperty(childProperty, key, childValue);
+                })
+                .map(o => attachPathToKeys(o, path))
+                .reduce((a, b) => ({ ...a,
+...b }), {});
+
+            if (Object.keys(mapProperties).length === 0) return {};
+            const mapParentProperty: InputProperty = {
+                name: property.name,
+                description: property.description,
+                type: property.type,
+                fieldConfigId: "group",
+                disabled: Boolean(property.ui?.disabled || property.ui?.readOnly)
+            };
+            return {
+                [path]: mapParentProperty,
+                ...mapProperties
+            } as Record<string, InputProperty>;
+        }
+    } else {
+        const fieldId = getFieldId(property);
+        if (!fieldId) {
+            console.warn(`No fieldId found for property ${path} with type ${property.type}`);
+            return {};
+        }
+        return {
+            [path]: getSimpleProperty(property)
+        };
+    }
+    return {};
+}
+
+// attach a path to every key in an object
+function attachPathToKeys(obj: Record<string, any>, path = ""): Record<string, any> {
+    return Object.entries(obj)
+        .map(([key, value]) => {
+            const fullKey = path ? `${path}.${key}` : key;
+            return { [fullKey]: value };
+        })
+        .reduce((a, b) => ({ ...a,
+...b }), {});
+}
+
+function getSimpleEnumValues(enumValues: EnumValues): string[] {
+    if (Array.isArray(enumValues))
+        return enumValues.map(v => String(v.id));
+    if (typeof enumValues === "object")
+        return Object.keys(enumValues);
+    throw Error("getSimpleEnumValues: Invalid enumValues");
+}

package/src/utils/strings_counter.ts

@@ -0,0 +1,22 @@
+import { EntityValues, Properties, Property } from "@rebasepro/types";
+
+export function countStringCharacters(values: EntityValues<any>, properties: Properties) {
+    let count = 0;
+
+    for (const key in values) {
+        const value = values[key];
+        const property: Property = properties[key];
+
+        if (property && !property.ui?.disabled) {
+            if (property.type === "string" || property.type === "number") {
+                count += String(value).length;
+            } else if (property.type === "array" && Array.isArray(value) && property.of && !Array.isArray(property.of) && property.of.type === "string") {
+                count += (value as string[]).reduce((acc, curr) => acc + (curr?.length ?? 0), 0);
+            } else if (property.type === "map" && property.properties && typeof value === "object") {
+                count += countStringCharacters(value, property.properties);
+            }
+        }
+    }
+
+    return count;
+}

package/src/utils/suggestions.ts

@@ -0,0 +1,6 @@
+export function getAppendableSuggestion(suggestion: string | number | undefined, value: any): string | undefined {
+    const suggestionIncludesValue = typeof suggestion === "string" && typeof value === "string" && suggestion.toLowerCase().trim().startsWith(value.toLowerCase().trim());
+    return (typeof value === "string" && suggestionIncludesValue)
+        ? suggestion.substring(suggestion.toLowerCase().trim().indexOf(value.toLowerCase().trim()) + value.trim().length)
+        : undefined;
+}

package/src/utils/values.ts

@@ -0,0 +1,12 @@
+export function flatMapEntityValues<M extends object>(values: M, path = ""): object {
+    if (!values) return {};
+    return Object.entries(values).flatMap(([key, value]) => {
+        const currentPath = path ? `${path}.${key}` : key;
+        if (typeof value === "object") {
+            return flatMapEntityValues(value, currentPath);
+        } else {
+            return { [currentPath]: value };
+        }
+    }).reduce((acc, curr) => ({ ...acc,
+...curr }), {})
+}

package/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />