@cardstack/boxel-cli 0.2.0-unstable.448 → 0.3.0-unstable.13

package/bundled-types/host-app/lib/matrix-classes/message.ts

@@ -0,0 +1,239 @@
+import { guidFor } from '@ember/object/internals';
+import { cached, tracked } from '@glimmer/tracking';
+
+import { TrackedArray } from 'tracked-built-ins';
+
+import { escapeHtmlOutsideCodeBlocks } from '@cardstack/runtime-common/helpers/html';
+import { markdownToHtml } from '@cardstack/runtime-common/marked-sync';
+
+import {
+  parseHtmlContent,
+  type HtmlTagGroup,
+} from '@cardstack/host/lib/formatted-message/utils';
+
+import type { FileDef } from 'https://cardstack.com/base/file-api';
+
+import type { RoomMember } from './member';
+
+import type MessageCodePatchResult from './message-code-patch-result';
+
+import type MessageCommand from './message-command';
+import type { EventStatus } from 'matrix-js-sdk';
+
+const ErrorMessage: Record<string, string> = {
+  ['M_TOO_LARGE']: 'Message is too large',
+};
+
+type RoomMessageInterface = RoomMessageRequired & RoomMessageOptional;
+
+interface RoomMessageRequired {
+  roomId: string;
+  author: RoomMember;
+  created: Date;
+  updated: Date;
+  body: string;
+  eventId: string;
+  status: EventStatus | null;
+}
+
+interface RoomMessageOptional {
+  transactionId?: string | null;
+  attachedCardIds?: string[] | null;
+  attachedFiles?: FileDef[];
+  isStreamingFinished?: boolean;
+  index?: number;
+  errorMessage?: string;
+  clientGeneratedId?: string | null;
+  reasoningContent?: string | null;
+  isDebugMessage?: boolean;
+  reloadBillingData?: boolean;
+  hasContinuation?: boolean;
+  continuationOf?: string | null;
+  agentId?: string;
+  isCodePatchCorrectness?: boolean;
+}
+
+export class Message implements RoomMessageInterface {
+  @tracked _body: string;
+  @tracked _reasoningContent?: string | null;
+  @tracked _commands: TrackedArray<MessageCommand>;
+  @tracked codePatchResults: TrackedArray<MessageCodePatchResult>;
+  @tracked created: Date;
+  @tracked _isStreamingFinished?: boolean;
+  @tracked _isCanceled?: boolean;
+  @tracked hasContinuation?: boolean;
+  @tracked continuedInMessage?: Message | null;
+  @tracked continuationOf?: string | null;
+
+  attachedCardIds?: string[] | null;
+  attachedFiles?: FileDef[];
+  attachedCardsAsFiles?: FileDef[];
+  attachedSkillCardIds?: string[] | null;
+  index?: number;
+  transactionId?: string | null;
+  @tracked errorMessage?: string;
+  clientGeneratedId?: string;
+  isDebugMessage?: boolean;
+  reloadBillingData?: boolean;
+  isCodePatchCorrectness?: boolean;
+
+  author: RoomMember;
+  @tracked status: EventStatus | null;
+  @tracked _updated: Date;
+  eventId: string;
+  roomId: string;
+  agentId?: string;
+
+  //This property is used for testing purpose
+  instanceId: string;
+
+  constructor(init: RoomMessageInterface) {
+    this._body = init.body;
+    this._reasoningContent = init.reasoningContent;
+    this._commands = new TrackedArray<MessageCommand>();
+    this.author = init.author;
+    this.eventId = init.eventId;
+    this.created = init.created;
+    this._updated = init.updated;
+    this.status = init.status;
+    this.roomId = init.roomId;
+    this.agentId = init.agentId;
+    this.attachedFiles = init.attachedFiles;
+    this.hasContinuation = init.hasContinuation;
+    this.continuationOf = init.continuationOf;
+    this._reasoningContent = init.reasoningContent;
+    this._commands = new TrackedArray<MessageCommand>();
+    this.codePatchResults = new TrackedArray<MessageCodePatchResult>();
+    this.instanceId = guidFor(this);
+    this.isCodePatchCorrectness = false;
+  }
+
+  get isRetryable() {
+    return (
+      this.errorMessage === undefined ||
+      (this.errorMessage && this.errorMessage !== ErrorMessage['M_TOO_LARGE'])
+    );
+  }
+
+  get reasoningContent(): string {
+    return [this._reasoningContent, this.continuedReasoningContent]
+      .filter(Boolean)
+      .join('');
+  }
+
+  setReasoningContent(reasoningContent: string | null) {
+    if (this._reasoningContent !== reasoningContent) {
+      this._reasoningContent = reasoningContent;
+    }
+  }
+
+  get continuedReasoningContent() {
+    return this.continuedInMessage?.reasoningContent ?? '';
+  }
+
+  get body(): string {
+    return [this._body, this.continuedBody].filter(Boolean).join('');
+  }
+
+  setBody(body: string) {
+    if (this._body !== body) {
+      this._body = body;
+    }
+  }
+
+  get continuedBody() {
+    return this.continuedInMessage?.body;
+  }
+
+  get commands(): MessageCommand[] {
+    return (this.continuedInMessage?.commands?.length ?? 0) > 0
+      ? this.continuedInMessage!.commands
+      : (this._commands ?? []);
+  }
+
+  setCommands(commands: MessageCommand[]) {
+    this._commands = new TrackedArray<MessageCommand>(commands);
+  }
+
+  get continuedCommands() {
+    return this.continuedInMessage?.commands;
+  }
+
+  setIsStreamingFinished(isStreamingFinished: boolean | undefined) {
+    if (this._isStreamingFinished !== isStreamingFinished) {
+      this._isStreamingFinished = isStreamingFinished;
+    }
+  }
+
+  setIsCanceled(isCanceled: boolean | undefined) {
+    if (this._isCanceled !== isCanceled) {
+      this._isCanceled = isCanceled;
+    }
+  }
+
+  get isCanceled(): boolean {
+    return this._isCanceled ?? false;
+  }
+
+  get isStreamingFinished(): boolean | undefined {
+    if (this.hasContinuation) {
+      return this.continuedInMessage?.isStreamingFinished ?? false;
+    }
+    return this._isStreamingFinished;
+  }
+
+  get isStreamingOfEventFinished(): boolean {
+    return this._isStreamingFinished === true;
+  }
+
+  get updated(): Date {
+    return this.continuedInMessage?.updated ?? this._updated;
+  }
+
+  setUpdated(updated: Date) {
+    if (this._updated.getTime() !== updated.getTime()) {
+      this._updated = updated;
+    }
+  }
+
+  @cached
+  get bodyHTML() {
+    // message is expected to be in markdown so we need to convert the markdown to html when the message is sent by the ai bot
+    if (!this.body) {
+      return this.body;
+    }
+    return markdownToHtml(escapeHtmlOutsideCodeBlocks(this.body), {
+      sanitize: false,
+      escapeHtmlInCodeBlocks: true,
+    });
+  }
+
+  /*
+    We are splitting the html into parts so that we can target the
+    code blocks (<pre> tags) and apply Monaco editor to them. Here is an
+    example of the html argument:
+
+    <p>Here is some code for you.</p>
+    <pre data-codeblock="javascript">const x = 1;</pre>
+    <p>I hope you like this code. But here is some more!</p>
+    <pre data-codeblock="javascript">const y = 2;</pre>
+    <p>Feel free to use it in your project.</p>
+
+    A drawback of this approach is that we can't render monaco editors for
+    code blocks that are nested inside other elements. We should make sure
+    our skills teach the model to respond with code blocks that are not nested
+    inside other elements.
+  */
+  @cached
+  get htmlParts(): HtmlTagGroup[] {
+    let htmlParts = parseHtmlContent(this.bodyHTML, this.roomId, this.eventId);
+    if (this._isCanceled) {
+      htmlParts.push({
+        type: 'non_pre_tag',
+        content: '<p style="font-weight: bold;">{Generation Cancelled}</p>',
+        codeData: null,
+      });
+    }
+    return htmlParts;
+  }
+}

package/bundled-types/host-app/lib/matrix-classes/room.ts

@@ -0,0 +1,174 @@
+import { cached, tracked } from '@glimmer/tracking';
+
+import EventEmitter from 'eventemitter3';
+
+import {
+  APP_BOXEL_ACTIVE_LLM,
+  APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
+  APP_BOXEL_LLM_MODE,
+  type LLMMode,
+} from '@cardstack/runtime-common/matrix-constants';
+
+import type { SerializedFile } from 'https://cardstack.com/base/file-api';
+import type {
+  ActiveLLMEvent,
+  MatrixEvent as DiscreteMatrixEvent,
+} from 'https://cardstack.com/base/matrix-event';
+
+import Mutex from '../mutex';
+
+import type { IEvent } from 'matrix-js-sdk';
+
+import type * as MatrixSDK from 'matrix-js-sdk';
+
+export type TempEvent = Partial<IEvent> & {
+  status: MatrixSDK.EventStatus | null;
+  error?: MatrixSDK.MatrixError;
+};
+
+export type SkillsConfig = {
+  enabledSkillCards: SerializedFile[];
+  disabledSkillCards: SerializedFile[];
+  commandDefinitions: SerializedFile[];
+};
+
+export default class Room {
+  @tracked private _events: DiscreteMatrixEvent[] = [];
+  @tracked private _roomState: MatrixSDK.RoomState | undefined;
+
+  constructor(public readonly roomId: string) {}
+
+  readonly mutex = new Mutex();
+  private readonly emitter = new EventEmitter();
+
+  waitForNextEvent(): Promise<void> {
+    return new Promise((resolve) => {
+      this.emitter.once('event.added', resolve);
+    });
+  }
+
+  get events() {
+    return this._events;
+  }
+
+  get name() {
+    return this._roomState?.events.get('m.room.name')?.get('')?.event.content
+      ?.name;
+  }
+
+  @cached
+  get memberIds(): string[] {
+    let memberEvents = (this._roomState?.events
+      .get('m.room.member')
+      ?.values() ?? []) as MatrixSDK.MatrixEvent[];
+    let memberIds = [...memberEvents.map((ev) => ev.event.state_key)];
+    return memberIds.filter((id) => id !== undefined) as string[];
+  }
+
+  hasActiveMember(userId: string): boolean {
+    let memberEvent = this._roomState?.events.get('m.room.member')?.get(userId);
+    let membership = memberEvent?.event.content?.membership;
+    return membership === 'join' || membership === 'invite';
+  }
+
+  notifyRoomStateUpdated(rs: MatrixSDK.RoomState) {
+    this._roomState = rs; // this is usually the same object, but some internal state has changed. This assignment kicks off reactivity.
+  }
+
+  get hasRoomState() {
+    return this._roomState !== undefined;
+  }
+
+  get skillsConfig() {
+    const content = this._roomState?.events
+      .get(APP_BOXEL_ROOM_SKILLS_EVENT_TYPE)
+      ?.get('')?.event.content ?? {
+      enabledSkillCards: [],
+      disabledSkillCards: [],
+      commandDefinitions: [],
+    };
+
+    return {
+      enabledSkillCards: content.enabledSkillCards
+        ? content.enabledSkillCards
+        : [],
+      disabledSkillCards: content.disabledSkillCards
+        ? content.disabledSkillCards
+        : [],
+      commandDefinitions: content.commandDefinitions ?? [],
+    } as {
+      enabledSkillCards: SerializedFile[];
+      disabledSkillCards: SerializedFile[];
+      commandDefinitions: SerializedFile[];
+    };
+  }
+
+  get activeLLMEventContent(): ActiveLLMEvent['content'] | undefined {
+    let event = this._roomState?.events
+      .get(APP_BOXEL_ACTIVE_LLM)
+      ?.get('')?.event;
+    return (event as ActiveLLMEvent)?.content;
+  }
+
+  get activeLLM() {
+    return this.activeLLMEventContent?.model;
+  }
+
+  get activeInputModalities(): string[] | undefined {
+    return this.activeLLMEventContent?.inputModalities;
+  }
+
+  get activeLLMMode(): LLMMode {
+    let event = this._roomState?.events.get(APP_BOXEL_LLM_MODE)?.get('')?.event;
+    return event && (event as any).content?.mode
+      ? ((event as any).content.mode as LLMMode)
+      : 'ask';
+  }
+
+  addEvent(event: TempEvent, oldEventId?: string) {
+    let { event_id: eventId, state_key: stateKey } = event;
+    eventId = eventId ?? stateKey; // room state may not necessary have an event ID
+
+    // If we are receiving an event which contains
+    // a data field, we may need to parse it
+    // because matrix doesn't support all json types
+    // Corresponding encoding is done in
+    // sendEvent in the matrix-service
+    if (event.content?.data) {
+      if (typeof event.content.data === 'string') {
+        event.content.data = JSON.parse(event.content.data);
+      }
+    }
+    eventId = eventId ?? stateKey; // room state may not necessary have an event ID
+    if (!eventId) {
+      throw new Error(
+        `bug: event ID is undefined for event ${JSON.stringify(
+          event,
+          null,
+          2,
+        )}`,
+      );
+    }
+    // duplicate events may be emitted from matrix, as well as the resolved room card might already contain this event
+    let matchingEvents = this._events.filter(
+      (e) => e.event_id === eventId || e.event_id === oldEventId,
+    );
+    if (matchingEvents.length > 1) {
+      throw new Error(
+        `bug: ${matchingEvents.length} events with the same event_id(s): ${eventId}, ${oldEventId}, expected a maximum of 1`,
+      );
+    }
+    if (matchingEvents.length === 0) {
+      this._events = [
+        ...(this._events ?? []),
+        event as unknown as DiscreteMatrixEvent,
+      ];
+      return;
+    }
+    let eventToReplace = matchingEvents[0];
+    let eventIndex = this._events.indexOf(eventToReplace);
+    this._events[eventIndex] = event as unknown as DiscreteMatrixEvent;
+    this._events = [...this._events];
+    this.emitter.emit('event.added');
+  }
+}

package/bundled-types/host-app/lib/matrix-utils.ts

@@ -0,0 +1,114 @@
+import difference from 'lodash/difference';
+
+export interface MatrixError {
+  data: {
+    errcode: string;
+    error: string;
+    retry_after_ms?: number;
+  };
+  httpStatus: number;
+  errcode: string;
+}
+export const eventDebounceMs = 100;
+
+export function isMatrixError(err: any): err is MatrixError {
+  return (
+    typeof err === 'object' &&
+    'data' in err &&
+    typeof err.data === 'object' &&
+    'errcode' in err.data &&
+    typeof err.data.errcode === 'string' &&
+    'error' in err.data &&
+    typeof err.data.error === 'string' &&
+    'httpStatus' in err &&
+    typeof err.httpStatus === 'number' &&
+    'errcode' in err &&
+    typeof err.errcode === 'string'
+  );
+}
+
+export function isValidPassword(password: string): boolean {
+  return password?.length >= 8;
+}
+
+export interface InteractiveAuth {
+  completed?: string[];
+  session: string;
+  flows: Flow[];
+  error?: string;
+  errcode?: string;
+}
+
+interface Flow {
+  stages: string[];
+}
+
+function isFlow(flow: any): flow is Flow {
+  if (
+    typeof flow === 'object' &&
+    'stages' in flow &&
+    Array.isArray(flow.stages)
+  ) {
+    if (flow.stages.find((s: any) => typeof s !== 'string')) {
+      return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+export function isInteractiveAuth(json: any): json is InteractiveAuth {
+  if (
+    typeof json === 'object' &&
+    'session' in json &&
+    typeof json.session === 'string' &&
+    'flows' in json &&
+    Array.isArray(json.flows)
+  ) {
+    if ('error' in json && typeof json.error !== 'string') {
+      return false;
+    }
+    if ('errcode' in json && typeof json.errcode !== 'string') {
+      return false;
+    }
+    if ('completed' in json && !Array.isArray(json.completed)) {
+      return false;
+    }
+    if (
+      'completed' in json &&
+      json.completed.length > 0 &&
+      json.completed.find((c: any) => typeof c !== 'string')
+    ) {
+      return false;
+    }
+
+    return json.flows.every((f: any) => isFlow(f));
+  }
+  return false;
+}
+
+export function nextUncompletedStage(authFlow: InteractiveAuth) {
+  if (authFlow.flows.length === 0) {
+    throw new Error(
+      `Completed all interactive auth stages but encountered unsuccessful interactive auth response: ${JSON.stringify(
+        authFlow,
+        null,
+        2,
+      )}`,
+    );
+  }
+  let remainingStages = difference(
+    authFlow.flows[0].stages,
+    authFlow.completed ?? [],
+  );
+  if (remainingStages.length === 0) {
+    throw new Error(
+      `Completed all interactive auth stages but encountered unsuccessful interactive auth response: ${JSON.stringify(
+        authFlow,
+        null,
+        2,
+      )}`,
+    );
+  }
+  return remainingStages[0];
+}

package/bundled-types/host-app/lib/mutex.ts

@@ -0,0 +1,62 @@
+import { buildWaiter } from '@ember/test-waiters';
+
+/*
+This class uses promises to implement a "mutex", which is a type of lock that can be used to
+ensure only one block of provided async code is run at a time, and that they are run in the
+order they were requested.
+
+It uses Ember's test waiter API to ensure that tests wait for any code waiting for the lock to complete execution.
+
+Usage example:
+
+// Usage example
+const mutex = new Mutex();
+
+async function criticalSection(num: number) {
+  await mutex.dispatch(async () => {
+    await new Promise((resolve) => setTimeout(resolve, Math.random() * 100));
+    console.log('Critical section ' + num);
+  });
+}
+criticalSection(1);
+criticalSection(2);
+criticalSection(3);
+
+// Output: (order is guaranteed despite the calling code not waiting to call the function three times and the random delays)
+// -> Critical section 1
+// -> Critical section 2
+// -> Critical section 3
+
+*/
+
+const waiter = buildWaiter('mutex:waiter');
+
+export default class Mutex {
+  private mutex = Promise.resolve();
+
+  // This method is private because the dispatch method is a less
+  // dangerous way to use the mutex. If we encounter a situation
+  // where we need to expose this method, we can do so.
+  private async lock(): Promise<() => void> {
+    let begin: (unlock: () => void) => void = (_unlock) => {};
+
+    this.mutex = this.mutex.then(() => {
+      return new Promise(begin);
+    });
+
+    return new Promise((res) => {
+      begin = res;
+    });
+  }
+
+  async dispatch<T>(fn: (() => T) | (() => Promise<T>)): Promise<T> {
+    const token = waiter.beginAsync();
+    const unlock = await this.lock();
+    try {
+      return await fn();
+    } finally {
+      unlock();
+      waiter.endAsync(token);
+    }
+  }
+}

package/bundled-types/host-app/lib/prerender-fetch-headers.ts

@@ -0,0 +1,46 @@
+import {
+  DURING_PRERENDER_HEADER,
+  X_BOXEL_CONSUMING_REALM_HEADER,
+  X_BOXEL_JOB_ID_HEADER,
+} from '@cardstack/runtime-common';
+
+// Set by the prerender server's `evaluateOnNewDocument` before the
+// SPA boots — `__boxelDuringPrerender = true`. Read here so the
+// realm-server fetch wrappers can attach the marker header on
+// search calls only, narrowly scoping the signal to the endpoints
+// that need it. See realm.ts:DURING_PRERENDER_HEADER for the full
+// chain.
+export function duringPrerenderHeaders(): Record<string, string> {
+  let flag = (globalThis as unknown as { __boxelDuringPrerender?: boolean })
+    .__boxelDuringPrerender;
+  return flag ? { [DURING_PRERENDER_HEADER]: '1' } : {};
+}
+
+// While rendering inside a prerender tab the render route writes
+// `__boxelConsumingRealm` with the URL of the realm whose card is
+// being rendered. Attach it to outbound search requests so the
+// realm-server's job-scoped cache layer can gate caching on the
+// indexer-traffic shape. Read each fetch (not cached at module
+// scope) so a tab that renders cards from multiple realms in
+// sequence sends the correct header per request. Returns an empty
+// object when the global is not set so non-prerender (live SPA)
+// fetches behave exactly as before.
+export function consumingRealmHeader(): Record<string, string> {
+  let r = (globalThis as unknown as { __boxelConsumingRealm?: string })
+    .__boxelConsumingRealm;
+  return r ? { [X_BOXEL_CONSUMING_REALM_HEADER]: r } : {};
+}
+
+// Companion to `consumingRealmHeader()`. The prerender server's
+// `prerenderVisitAttempt` injects `__boxelJobId` onto the page
+// before transitioning into the render route — see
+// `packages/realm-server/prerender/render-runner.ts`. Read it on
+// each fetch (not module-scope-cached) so a page reused across
+// multiple visits picks up the current visit's job id. Outside a
+// prerender tab the global is undefined and we send no header, so
+// user / API callers continue to bypass the realm-server's
+// job-scoped cache.
+export function jobIdHeader(): Record<string, string> {
+  let j = (globalThis as unknown as { __boxelJobId?: string }).__boxelJobId;
+  return j ? { [X_BOXEL_JOB_ID_HEADER]: j } : {};
+}

package/bundled-types/host-app/lib/prerender-util.ts

@@ -0,0 +1,31 @@
+import {
+  visitModuleDeps,
+  resolveCardReference,
+  type LooseCardResource,
+  type Loader,
+} from '@cardstack/runtime-common';
+
+export function directModuleDeps(
+  resource: LooseCardResource,
+  instanceURL: URL,
+): string[] {
+  let result: string[] = [];
+  visitModuleDeps(resource, (moduleURL) => {
+    result.push(resolveCardReference(moduleURL, instanceURL));
+  });
+  return result;
+}
+
+export async function transitiveModuleDeps(
+  directDeps: string[],
+  loader: Loader,
+) {
+  return new Set([
+    ...directDeps,
+    ...(
+      await Promise.all(
+        directDeps.map((moduleDep) => loader.getConsumedModules(moduleDep)),
+      )
+    ).flat(),
+  ]);
+}

package/bundled-types/host-app/lib/public-path.ts

@@ -0,0 +1 @@
+(globalThis as any).__bootStart = performance.now();

package/bundled-types/host-app/lib/random-name.ts

@@ -0,0 +1,26 @@
+import {
+  adjectives,
+  animals,
+  NumberDictionary,
+  uniqueNamesGenerator,
+} from 'unique-names-generator';
+
+const numberDictionary = NumberDictionary.generate({ min: 0, max: 99 });
+
+export function generateRandomWorkspaceName(): string {
+  let includeNumber = Math.random() < 0.7;
+  let dictionaries = includeNumber
+    ? [adjectives, animals, numberDictionary]
+    : [adjectives, animals];
+
+  let slug = uniqueNamesGenerator({
+    dictionaries,
+    separator: '-',
+    style: 'lowerCase',
+  });
+  return slug
+    .split('-')
+    .filter(Boolean)
+    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
+    .join(' ');
+}