@dxos/plugin-script 0.8.1-staging.391c573 → 0.8.1-staging.97aedb1

package/src/meta.ts

@@ -6,7 +6,7 @@ import { type PluginMeta } from '@dxos/app-framework';
 
 export const SCRIPT_PLUGIN = 'dxos.org/plugin/script';
 
-export const meta = {
+export const meta: PluginMeta = {
   id: SCRIPT_PLUGIN,
   name: 'Scripts',
   description:
@@ -15,4 +15,9 @@ export const meta = {
   source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/experimental/plugin-explorer',
   tags: ['experimental'],
   screenshots: ['https://dxos.network/plugin-details-scripts-dark.png'],
-} satisfies PluginMeta;
+};
+
+// TODO(ZaymonFC): Configure by scopes?
+export const defaultScriptsForIntegration: Record<string, string[]> = {
+  'gmail.com': ['dxos.org/script/gmail'],
+};

package/src/templates/chess.ts

@@ -1,3 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
 // @ts-ignore
 import { defineFunction, S } from 'dxos:functions';
 // @ts-ignore

package/src/templates/data-generator.ts

@@ -0,0 +1,73 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import {
+  EchoObject,
+  defineFunction,
+  S,
+  createStatic,
+  Expando,
+  TypedObject,
+  Ref,
+  LabelAnnotationId,
+  // @ts-ignore
+} from 'dxos:functions';
+// @ts-ignore
+import { randParagraph } from 'https://esm.sh/@ngneat/falso@7.1.1';
+
+export class TextType extends TypedObject({ typename: 'dxos.org/type/Text', version: '0.1.0' })({
+  content: S.String,
+}) {}
+
+export const DocumentSchema = S.Struct({
+  name: S.optional(S.String),
+  fallbackName: S.optional(S.String),
+  content: Ref(TextType),
+  threads: S.mutable(S.Array(Ref(Expando))),
+}).annotations({
+  [LabelAnnotationId]: ['name', 'fallbackName'],
+});
+
+export const DocumentType = DocumentSchema.pipe(EchoObject('dxos.org/type/Document', '0.1.0'));
+export type DocumentType = S.Schema.Type<typeof DocumentType>;
+
+export default defineFunction({
+  inputSchema: S.Struct({
+    documentAmount: S.optional(S.Number),
+    textSize: S.optional(S.Number),
+    mutationAmount: S.optional(S.Number),
+  }),
+
+  outputSchema: S.Struct({}),
+
+  handler: async ({ event: { documentAmount = 10, textSize = 10, mutationAmount = 0 }, context: { space } }: any) => {
+    await space.db.graph.schemaRegistry.addSchema([Expando]);
+
+    const objects = [];
+    // const rootCollection = await space.properties[COLLECTION_TYPENAME]?.load();
+    for (let i = 0; i < documentAmount; i++) {
+      const obj = space.db.add(
+        createStatic(Expando, {
+          name: 'scriptGenerated',
+          content: randParagraph({ length: textSize }),
+        }),
+      );
+      // rootCollection.objects.push(obj);
+      objects.push(obj);
+    }
+
+    for (let i = 0; i < mutationAmount; i++) {
+      const obj = objects[Math.floor(Math.random() * objects.length)];
+      obj.content = randParagraph({ length: textSize });
+    }
+
+    await space.db.flush();
+    // TODO(mykola): This seems to halp with propagating data to space.
+    for (const obj of objects) {
+      await space.db.query({ id: obj.id }).first();
+      console.log('obj', obj.id);
+    }
+    return 'OK';
+  },
+});

package/src/templates/discord.ts

@@ -19,7 +19,7 @@ const MessageSchema = S.Struct({
   from: S.String,
   created: S.String,
   content: S.String,
-}).pipe(EchoObject('example.com/type/Message', '0.1.0'));
+}).pipe(EchoObject({ typename: 'example.com/type/Message', version: '0.1.0' }));
 
 const DEFAULT_AFTER = 1704067200; // 2024-01-01
 

package/src/templates/forex-effect.ts

@@ -33,9 +33,10 @@ export default defineFunction({
         HttpClient.execute,
         Effect.flatMap((res: any) => res.json),
         Effect.timeout('1 second'),
-        Effect.retry(Schedule.exponential(1000).pipe(Schedule.compose(Schedule.recurs(3)))),
+        Effect.retry(Schedule.exponential(1_000).pipe(Schedule.compose(Schedule.recurs(3)))),
         Effect.scoped,
       );
+
       return res.data.rates[to].toString();
     }).pipe(Effect.provide(FetchHttpClient.layer)),
 });

package/src/templates/gmail.ts

@@ -0,0 +1,244 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+// @ts-ignore
+import { createStatic, defineFunction, EchoObject, Filter, ObjectId, S } from 'dxos:functions';
+import {
+  HttpClient,
+  HttpClientRequest,
+  FetchHttpClient,
+  // @ts-ignore
+} from 'https://esm.sh/@effect/platform@0.77.2?deps=effect@3.13.3';
+// @ts-ignore
+import { pipe, Chunk, Effect, Ref, Schedule, Stream } from 'https://esm.sh/effect@3.13.3';
+
+// TODO(ZaymonFC): Calculate this dynamically and expose a parameter.
+const DEFAULT_AFTER = '2025-01-01';
+
+export default defineFunction({
+  inputSchema: S.Struct({
+    // TODO(wittjosiah): Remove. This is used to provide a terminal for a cron trigger.
+    tick: S.optional(S.String),
+    userId: S.optional(S.String),
+    after: S.optional(S.Union(S.Number, S.String)),
+    pageSize: S.optional(S.Number),
+    mailboxId: S.String,
+  }),
+
+  outputSchema: S.Struct({
+    newMessages: S.Number,
+  }),
+
+  handler: ({
+    event: {
+      data: { mailboxId, userId = 'me', after = DEFAULT_AFTER, pageSize = 100 },
+    },
+    context: { space },
+  }: any) =>
+    Effect.gen(function* () {
+      const { token } = yield* Effect.tryPromise({
+        try: () => space.db.query(Filter.typename('dxos.org/type/AccessToken', { source: 'gmail.com' })).first(),
+        catch: (e: any) => e,
+      });
+
+      const makeRequest = (path: string) =>
+        pipe(
+          path,
+          HttpClientRequest.get,
+          HttpClientRequest.setHeaders({ Authorization: `Bearer ${token}`, Accept: 'application/json' }),
+          HttpClient.execute,
+          Effect.flatMap((res: any) => res.json),
+          Effect.timeout('1 second'),
+          Effect.retry(Schedule.exponential(1000).pipe(Schedule.compose(Schedule.recurs(3)))),
+          Effect.scoped,
+        );
+
+      const listMessages = (userId: string, q: string, pageSize: number, pageToken: string | null) => {
+        const url = `https://gmail.googleapis.com/gmail/v1/users/${userId}/messages?maxResults=${pageSize}&q=${q}`;
+        return makeRequest(pageToken ? `${url}&pageToken=${pageToken}` : url);
+      };
+
+      const getMessage = (userId: string, id: string) =>
+        makeRequest(`https://gmail.googleapis.com/gmail/v1/users/${userId}/messages/${id}`);
+
+      const mailbox = yield* Effect.tryPromise({
+        try: () => space.db.query({ id: mailboxId }).first(),
+        catch: (e: any) => e,
+      });
+      const { objects } = yield* Effect.tryPromise({
+        try: () => space.queues.queryQueue(mailbox.queue.dxn),
+        catch: (e: any) => e,
+      });
+      const newMessages = yield* Ref.make([]);
+      const nextPage = yield* Ref.make(null);
+
+      do {
+        const last = objects.at(-1);
+        const q = last
+          ? `in:inbox after:${Math.floor(new Date(last.created).getTime() / 1000)}`
+          : `in:inbox after:${after}`;
+        const pageToken = yield* Ref.get(nextPage);
+        const { messages, nextPageToken } = yield* listMessages(userId, q, pageSize, pageToken);
+        yield* Ref.update(nextPage, () => nextPageToken);
+        for (const message of messages) {
+          const messageDetails = yield* getMessage(userId, message.id);
+          const created = new Date(parseInt(messageDetails.internalDate)).toISOString();
+          const from = messageDetails.payload.headers.find((h: any) => h.name === 'From');
+          const sender = from && parseEmailString(from.value);
+          // TODO(wittjosiah): Improve parsing of email contents.
+          const content =
+            messageDetails.payload.body?.data ??
+            messageDetails.payload.parts?.find((p: any) => p.mimeType === 'text/plain')?.body.data;
+          // Skip the message if content or sender is missing.
+          // Skip the message if it's the same as the last message.
+          // TODO(wittjosiah): This comparison should be done via foreignId probably.
+          if (!content || !sender || created === last?.created) {
+            continue;
+          }
+          const subject = messageDetails.payload.headers.find((h: any) => h.name === 'Subject')?.value;
+          const object = createStatic(MessageType, {
+            id: ObjectId.random(),
+            created,
+            sender,
+            blocks: [
+              {
+                type: 'text',
+                text: Buffer.from(content, 'base64').toString('utf-8'),
+              },
+            ],
+            properties: {
+              subject,
+              threadId: message.threadId,
+            },
+          });
+          // TODO(wittjosiah): Set foreignId in object meta.
+          yield* Ref.update(newMessages, (n: any) => [object, ...n]);
+        }
+      } while (yield* Ref.get(nextPage));
+
+      const queueMessages = yield* Ref.get(newMessages);
+      if (queueMessages.length > 0) {
+        yield* pipe(
+          queueMessages,
+          Stream.fromIterable,
+          Stream.grouped(10),
+          Stream.flatMap((batch: any) =>
+            Effect.tryPromise({
+              try: () => space.queues.insertIntoQueue(mailbox.queue.dxn, Chunk.toReadonlyArray(batch)),
+              catch: (e: any) => e,
+            }),
+          ),
+          Stream.runDrain,
+        );
+      }
+
+      return { newMessages: queueMessages.length };
+    }).pipe(Effect.provide(FetchHttpClient.layer)),
+});
+
+/**
+ * Parses an email string in the format "Name <email@example.com>" into separate name and email components.
+ */
+const parseEmailString = (emailString: string): { name?: string; email: string } | undefined => {
+  const match = emailString.match(/^([^<]+?)\s*<([^>]+@[^>]+)>$/);
+  if (match) {
+    const [, name, email] = match;
+    return {
+      name: name.trim(),
+      email: email.trim(),
+    };
+  }
+
+  return undefined;
+};
+
+// TODO(wittjosiah): These schemas should be imported from @dxos/schema.
+
+//
+// Schemas
+//
+
+const ActorRoles = ['user', 'assistant'] as const;
+const ActorRole = S.Literal(...ActorRoles);
+type ActorRole = S.Schema.Type<typeof ActorRole>;
+
+const ActorSchema = S.Struct({
+  identityKey: S.optional(S.String),
+  email: S.optional(S.String),
+  name: S.optional(S.String),
+  role: S.optional(ActorRole),
+});
+
+const AbstractContentBlock = S.Struct({
+  pending: S.optional(S.Boolean),
+});
+type AbstractContentBlock = S.Schema.Type<typeof AbstractContentBlock>;
+const TextContentBlock = S.extend(
+  AbstractContentBlock,
+  S.Struct({
+    type: S.Literal('text'),
+    disposition: S.optional(S.String),
+    text: S.String,
+  }),
+).pipe(S.mutable);
+type TextContentBlock = S.Schema.Type<typeof TextContentBlock>;
+const JsonContentBlock = S.extend(
+  AbstractContentBlock,
+  S.Struct({
+    type: S.Literal('json'),
+    disposition: S.optional(S.String),
+    data: S.String,
+  }),
+).pipe(S.mutable);
+type JsonContentBlock = S.Schema.Type<typeof JsonContentBlock>;
+const Base64ImageSource = S.Struct({
+  type: S.Literal('base64'),
+  mediaType: S.String,
+  data: S.String,
+}).pipe(S.mutable);
+const HttpImageSource = S.Struct({
+  type: S.Literal('http'),
+  url: S.String,
+}).pipe(S.mutable);
+const ImageSource = S.Union(Base64ImageSource, HttpImageSource);
+type ImageSource = S.Schema.Type<typeof ImageSource>;
+const ImageContentBlock = S.extend(
+  AbstractContentBlock,
+  S.Struct({
+    type: S.Literal('image'),
+    id: S.optional(S.String),
+    source: S.optional(ImageSource),
+  }),
+).pipe(S.mutable);
+type ImageContentBlock = S.Schema.Type<typeof ImageContentBlock>;
+const ReferenceContentBlock = S.extend(
+  AbstractContentBlock,
+  S.Struct({
+    type: S.Literal('reference'),
+    reference: S.Any,
+  }),
+).pipe(S.mutable);
+type ReferenceContentBlock = S.Schema.Type<typeof ReferenceContentBlock>;
+const MessageContentBlock = S.Union(TextContentBlock, JsonContentBlock, ImageContentBlock, ReferenceContentBlock);
+
+const MessageType = S.Struct({
+  id: ObjectId,
+  created: S.String.annotations({
+    description: 'ISO date string when the message was sent.',
+  }),
+  sender: ActorSchema.annotations({
+    description: 'Identity of the message sender.',
+  }),
+  blocks: S.Array(MessageContentBlock).annotations({
+    description: 'Contents of the message.',
+  }),
+  properties: S.optional(
+    S.mutable(
+      S.Record({ key: S.String, value: S.Any }).annotations({
+        description: 'Custom properties for specific message types (e.g. attention context, email subject, etc.).',
+      }),
+    ),
+  ),
+}).pipe(EchoObject({ typename: 'dxos.org/type/Message', version: '0.1.0' }));
+type MessageType = S.Schema.Type<typeof MessageType>;

package/src/templates/index.ts

@@ -4,11 +4,13 @@
 
 import chat from './chat.ts?raw';
 import chess from './chess.ts?raw';
+import dataGenerator from './data-generator.ts?raw';
 import discord from './discord.ts?raw';
 import echo from './echo.ts?raw';
 import email from './email.ts?raw';
 import forexEffect from './forex-effect.ts?raw';
 import forex from './forex.ts?raw';
+import gmail from './gmail.ts?raw';
 
 const removeHeader = (str: string) => str.split('\n').slice(4).join('\n');
 
@@ -59,4 +61,16 @@ export const templates: Template[] = [
     source: removeHeader(discord),
     presetId: 'dxos.org/function/discord',
   },
+  {
+    id: 'dxos.org/script/gmail',
+    name: 'Gmail',
+    source: removeHeader(gmail),
+    presetId: 'dxos.org/function/gmail',
+  },
+  {
+    id: 'dxos.org/script/data-generator',
+    name: 'Data Generator',
+    source: removeHeader(dataGenerator),
+    presetId: 'dxos.org/function/data-generator',
+  },
 ];

package/src/translations.ts

@@ -11,14 +11,12 @@ export default [
     'en-US': {
       [ScriptType.typename]: {
         'typename label': 'Script',
+        'object name placeholder': 'New script',
       },
       [SCRIPT_PLUGIN]: {
         'plugin name': 'Scripts',
-        'object title placeholder': 'New script',
-        'create object label': 'Create script',
-        'create stack section label': 'Create script',
         'description label': 'Description',
-        'description placeholder': 'Describe what the script does.',
+        'description placeholder': 'Enter description',
         'binding placeholder': 'Function name',
         'function request placeholder': 'Function request',
         'deploy label': 'Deploy',
@@ -33,10 +31,10 @@ export default [
         'remote function settings heading': 'Remote Function',
         'function url label': 'Invocation URL',
         'function binding label': 'Function binding',
-        'function binding placeholder': 'BINDING',
+        'function binding placeholder': 'Enter binding name (e.g., NAME)',
         'script publish settings label': 'Publishing',
         'script publish settings description': 'Scripts can be shared publicly as Github gists.',
-        'no github token label': 'This space has no Github access token available.',
+        'no github token label': 'This space has no GitHub access token available.',
         'open token manager label': 'Open Token Manager',
         'publish label': 'Publish',
         'publishing label': 'Publishing',
@@ -45,12 +43,32 @@ export default [
         'settings editor input mode default label': 'Default',
         'settings editor input mode vim label': 'Vim',
         'settings editor input mode vscode label': 'VS Code',
-        'view group label': 'View',
-        'view editor label': 'Editor',
-        'view split label': 'Split',
-        'view logs label': 'Logs',
         'template select group label': 'Select template',
         'function panel label': 'Debug Function',
+        'script test label': 'Test',
+        'script logs label': 'Logs',
+        'deployment dialog title': 'Default Scripts',
+        'deployment dialog scripts found message_one':
+          'We found the following script template for your new integration:',
+        'deployment dialog scripts found message_other':
+          'We found the following script templates for your new integration:',
+        'deployment dialog deploy functions button label_one': 'Create and deploy script',
+        'deployment dialog deploy functions button label_other': 'Create and deploy scripts',
+        'deployment dialog deploy functions pending button label_one': 'Deploying script',
+        'deployment dialog deploy functions pending button label_other': 'Deploying scripts',
+        'deployment dialog skip button label': 'Skip',
+        'script deployment toast label_one': 'Script deployed',
+        'script deployment toast label_other': 'Scripts deployed',
+        'script deployment toast description_one': 'Your script has been deployed successfully.',
+        'script deployment toast description_other': 'Your scripts have been deployed successfully.',
+        'script deployment toast close label_one': 'Close',
+        'script deployment toast close label_other': 'Close',
+        'script deployment error toast label_one': 'Script deployment failed',
+        'script deployment error toast label_other': 'Scripts deployment failed',
+        'script deployment error toast description_one': 'There was an error deploying your script.',
+        'script deployment error toast description_other': 'There was an error deploying your scripts.',
+        'script deployment error toast close label_one': 'Close',
+        'script deployment error toast close label_other': 'Close',
       },
     },
   },

package/src/types.ts

@@ -11,6 +11,9 @@ import { EditorInputMode } from '@dxos/react-ui-editor';
 
 import { SCRIPT_PLUGIN } from './meta';
 
+// TODO(burdon): Standardize export (e.g., @dxos/plugin-script/meta)?
+export { SCRIPT_PLUGIN };
+
 export namespace ScriptAction {
   const SCRIPT_ACTION = `${SCRIPT_PLUGIN}/action`;
 
@@ -18,6 +21,7 @@ export namespace ScriptAction {
     name: S.optional(S.String),
     // TODO(wittjosiah): Placeholder annotation?
     gistUrl: S.optional(S.String.annotations({ [TitleAnnotationId]: 'Import from Gist (url)' })),
+    initialTemplateId: S.optional(S.String),
   });
 
   export type CreateScriptProps = S.Schema.Type<typeof CreateScriptSchema>;

package/src/util/deploy.ts

@@ -0,0 +1,126 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Client } from '@dxos/client';
+import {
+  FunctionType,
+  type ScriptType,
+  getUserFunctionUrlInMetadata,
+  incrementSemverPatch,
+  setUserFunctionUrlInMetadata,
+  uploadWorkerFunction,
+} from '@dxos/functions';
+import { Bundler } from '@dxos/functions/bundler';
+import { log } from '@dxos/log';
+import { create, getMeta, makeRef, type Space } from '@dxos/react-client/echo';
+
+import { updateFunctionMetadata } from './functions';
+
+export const isScriptDeployed = ({ script, fn }: { script: ScriptType; fn: any }): boolean => {
+  const existingFunctionUrl = fn && getUserFunctionUrlInMetadata(getMeta(fn));
+  return Boolean(existingFunctionUrl) && !script.changed;
+};
+
+type DeployScriptProps = {
+  script: ScriptType;
+  client: Client;
+  space: Space;
+  fn?: FunctionType;
+  existingFunctionUrl?: string;
+};
+
+type DeployScriptResult = { success: boolean; error?: Error; functionUrl?: string };
+
+/**
+ * Deploy a script to a space, handling bundling and uploading to the FaaS infrastructure.
+ */
+export const deployScript = async ({
+  script,
+  client,
+  space,
+  fn,
+  existingFunctionUrl,
+}: DeployScriptProps): Promise<DeployScriptResult> => {
+  const validationError = validateDeployInputs(script, space);
+  if (validationError) {
+    return { success: false, error: validationError };
+  }
+
+  try {
+    const existingFunctionId = extractFunctionId(existingFunctionUrl);
+
+    const { bundle, error } = await bundleScript(script.source!.target!.content);
+    if (error || !bundle) {
+      throw error || new Error('Bundle creation failed');
+    }
+
+    const { functionId, version, meta } = await uploadWorkerFunction({
+      client,
+      spaceId: space.id,
+      version: fn ? incrementSemverPatch(fn.version) : '0.0.1',
+      functionId: existingFunctionId,
+      source: bundle,
+    });
+
+    if (functionId === undefined || version === undefined) {
+      throw new Error(`Upload didn't return expected data: ${JSON.stringify({ functionId, version })}`);
+    }
+
+    const storedFunction = createOrUpdateFunctionInSpace(space, fn, script, functionId, version);
+    script.changed = false;
+    updateFunctionMetadata(script, storedFunction, meta, functionId);
+
+    const functionUrl = makeFunctionUrl(space.id, functionId);
+    setUserFunctionUrlInMetadata(getMeta(storedFunction), functionUrl);
+
+    return { success: true, functionUrl };
+  } catch (err: any) {
+    log.catch(err);
+    return { success: false, error: err };
+  }
+};
+
+/**
+ * Validate inputs for script deployment.
+ */
+const validateDeployInputs = (script: ScriptType, space: Space): Error | null => {
+  if (!script.source || !space) {
+    return new Error('Script source or space not available');
+  }
+  return null;
+};
+
+const extractFunctionId = (functionUrl?: string): string | undefined => {
+  return functionUrl?.split('/').at(-1);
+};
+
+const bundleScript = async (source: string): Promise<{ bundle?: string; error?: Error }> => {
+  const bundler = new Bundler({ platform: 'browser', sandboxedModules: [], remoteModules: {} });
+  const buildResult = await bundler.bundle({ source });
+
+  if (buildResult.error || !buildResult.bundle) {
+    return { error: buildResult.error || new Error('Bundle creation failed') };
+  }
+
+  return { bundle: buildResult.bundle };
+};
+
+const createOrUpdateFunctionInSpace = (
+  space: Space,
+  fn: FunctionType | undefined,
+  script: ScriptType,
+  functionId: string,
+  version: string,
+): FunctionType => {
+  if (fn) {
+    fn.version = version;
+    return fn;
+  } else {
+    return space.db.add(create(FunctionType, { name: functionId, version, source: makeRef(script) }));
+  }
+};
+
+const makeFunctionUrl = (spaceId: string, functionId: string): string => {
+  return `/${spaceId}/${functionId}`;
+};

package/src/util/functions.ts

@@ -0,0 +1,53 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type ScriptType, getInvocationUrl, getUserFunctionUrlInMetadata } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { getMeta, getSpace } from '@dxos/react-client/echo';
+
+/**
+ * Get the function URL for a given script and client configuration
+ */
+export const getFunctionUrl = ({
+  script,
+  fn,
+  edgeUrl,
+}: {
+  script: ScriptType;
+  fn: any;
+  edgeUrl: string;
+}): string | undefined => {
+  const space = getSpace(script);
+  const existingFunctionUrl = fn && getUserFunctionUrlInMetadata(getMeta(fn));
+
+  if (!existingFunctionUrl) {
+    return undefined;
+  }
+
+  return getInvocationUrl(existingFunctionUrl, edgeUrl, {
+    spaceId: space?.id,
+  });
+};
+
+export const updateFunctionMetadata = (script: ScriptType, storedFunction: any, meta: any, functionId: string) => {
+  if (script.description !== undefined && script.description.trim() !== '') {
+    storedFunction.description = script.description;
+  } else if (meta.description) {
+    storedFunction.description = meta.description;
+  } else {
+    log.verbose('no description in function metadata', { functionId });
+  }
+
+  if (meta.inputSchema) {
+    storedFunction.inputSchema = meta.inputSchema;
+  } else {
+    log.verbose('no input schema in function metadata', { functionId });
+  }
+
+  if (meta.outputSchema) {
+    storedFunction.outputSchema = meta.outputSchema;
+  } else {
+    log.verbose('no output schema in function metadata', { functionId });
+  }
+};

package/src/util/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './deploy';
+export * from './functions';