@promptbook/cli 0.114.0-2 → 0.114.0-21

package/apps/agents-server/src/utils/agentProjects/resolveAgentChatProjectReferences.ts

@@ -0,0 +1,25 @@
+import { resolveAgentProjectsAccess, type AgentProjectsAccessResolution } from './agentProjectAccess';
+import type { AgentProjectReferenceInfo } from './AgentProjectReferenceInfo';
+import { listAgentProjectChatReferences } from './listAgentProjectChatReferences';
+
+/**
+ * Resolves the compact project references one chat surface may render as project chips.
+ *
+ * Every surface showing a chat — the agent profile, the chat page, and the action refreshing an open
+ * chat — gates the projects behind the same overview permission, so the decision lives here only.
+ *
+ * @param options - Owning agent and its already resolved project access, when the caller has it.
+ * @returns Display-only project references safe to send to the browser.
+ */
+export async function resolveAgentChatProjectReferences(options: {
+    readonly agentPermanentId: string;
+    readonly projectsAccess?: AgentProjectsAccessResolution;
+}): Promise<ReadonlyArray<AgentProjectReferenceInfo>> {
+    const projectsAccess = options.projectsAccess || (await resolveAgentProjectsAccess(options.agentPermanentId));
+
+    if (!projectsAccess.isProjectOverviewVisible) {
+        return [];
+    }
+
+    return await listAgentProjectChatReferences(options.agentPermanentId);
+}

package/apps/agents-server/src/utils/agentProjects/resolveAgentProjectFaviconRelativePath.ts

@@ -0,0 +1,108 @@
+import { stat } from 'fs/promises';
+import { join } from 'path';
+import { AGENT_PROJECT_CONVENTIONAL_FAVICON_RELATIVE_PATHS } from './agentProjectFileNames';
+import { isSafeAgentProjectPathSegment } from './agentProjectsPaths';
+import { isMissingPathError } from './isMissingPathError';
+import { resolveAgentProjectFileContentType } from './resolveAgentProjectFileContentType';
+
+/**
+ * Matches an icon reference which is not served from the project folder itself,
+ * for example `https://example.com/favicon.ico`, `//example.com/favicon.ico` or a `data:` URI.
+ */
+const EXTERNAL_ICON_HREF_REGEX = /^([a-z][a-z0-9+.-]*:|\/\/)/i;
+
+/**
+ * Content type prefix every usable project icon file must have.
+ */
+const ICON_CONTENT_TYPE_PREFIX = 'image/';
+
+/**
+ * Resolves the icon of one project as a path relative to the project root.
+ *
+ * The icon linked by the project `index.html` wins, the conventional favicon locations are searched
+ * afterwards. Only files which really exist inside the project folder are returned, so the resolved
+ * path can always be served through the project file route.
+ *
+ * @param options - Project directory and the icon href linked by its `index.html`.
+ * @returns Project-relative icon path using `/` separators, or `null` when the project has no icon.
+ */
+export async function resolveAgentProjectFaviconRelativePath(options: {
+    readonly projectPath: string;
+    readonly indexHtmlFaviconHref: string | null;
+}): Promise<string | null> {
+    const linkedFaviconRelativePath = resolveLocalIconRelativePath(options.indexHtmlFaviconHref);
+    const faviconRelativePathCandidates = [
+        ...(linkedFaviconRelativePath === null ? [] : [linkedFaviconRelativePath]),
+        ...AGENT_PROJECT_CONVENTIONAL_FAVICON_RELATIVE_PATHS,
+    ];
+
+    for (const faviconRelativePathCandidate of faviconRelativePathCandidates) {
+        if (await isExistingProjectIconFile(options.projectPath, faviconRelativePathCandidate)) {
+            return faviconRelativePathCandidate;
+        }
+    }
+
+    return null;
+}
+
+/**
+ * Normalizes one icon href written in a project `index.html` into a project-relative path.
+ *
+ * The href is written by the agent and therefore untrusted — an icon hosted outside of the project
+ * folder and any path escaping the project root are refused instead of being reported as an error.
+ *
+ * @param iconHref - Icon href exactly as written in the document.
+ * @returns Project-relative icon path using `/` separators, or `null` when the href is not usable.
+ */
+function resolveLocalIconRelativePath(iconHref: string | null): string | null {
+    const trimmedIconHref = (iconHref || '').trim();
+
+    if (!trimmedIconHref || EXTERNAL_ICON_HREF_REGEX.test(trimmedIconHref)) {
+        return null;
+    }
+
+    const iconPath = trimmedIconHref.split(/[?#]/u)[0] || '';
+    let iconPathSegments: ReadonlyArray<string>;
+
+    try {
+        iconPathSegments = iconPath
+            .split('/')
+            .filter((iconPathSegment) => iconPathSegment !== '' && iconPathSegment !== '.')
+            .map((iconPathSegment) => decodeURIComponent(iconPathSegment));
+    } catch {
+        return null;
+    }
+
+    if (iconPathSegments.length === 0 || !iconPathSegments.every(isSafeAgentProjectPathSegment)) {
+        return null;
+    }
+
+    return iconPathSegments.join('/');
+}
+
+/**
+ * Returns whether one project-relative path points to an image file inside the project.
+ *
+ * @param projectPath - Absolute path of the project directory.
+ * @param iconRelativePath - Project-relative icon path using `/` separators.
+ * @returns `true` when the path is an existing image file.
+ */
+async function isExistingProjectIconFile(projectPath: string, iconRelativePath: string): Promise<boolean> {
+    const iconPathSegments = iconRelativePath.split('/');
+    const iconFileName = iconPathSegments[iconPathSegments.length - 1] || '';
+
+    if (!resolveAgentProjectFileContentType(iconFileName).startsWith(ICON_CONTENT_TYPE_PREFIX)) {
+        return false;
+    }
+
+    try {
+        const iconStats = await stat(join(projectPath, ...iconPathSegments));
+        return iconStats.isFile();
+    } catch (error) {
+        if (isMissingPathError(error)) {
+            return false;
+        }
+
+        throw error;
+    }
+}

package/apps/agents-server/src/utils/agentProjects/resolveAgentProjectInfo.ts

@@ -5,8 +5,10 @@ import type { AgentProjectInfo } from './AgentProjectInfo';
 import { resolveAgentProjectsRootPath, resolveSafeAgentProjectPath } from './agentProjectsPaths';
 import { isMissingPathError } from './isMissingPathError';
 import { measureDirectoryUsage } from './measureDirectoryUsage';
+import { readAgentProjectIndexHtmlProfile } from './readAgentProjectIndexHtmlProfile';
 import { readAgentProjectReadme } from './readAgentProjectReadme';
-import { resolveAgentProjectReadmeProfile } from './resolveAgentProjectReadmeProfile';
+import { resolveAgentProjectFaviconRelativePath } from './resolveAgentProjectFaviconRelativePath';
+import { resolveAgentProjectProfile } from './resolveAgentProjectProfile';
 
 /**
  * Builds metadata for one project directory.
@@ -20,14 +22,20 @@ export async function createAgentProjectInfo(
     projectDirectoryName: string,
 ): Promise<AgentProjectInfo> {
     const projectPath = join(projectsRootPath, projectDirectoryName);
-    const [projectUsage, isGitRepository, readme] = await Promise.all([
+    const [projectUsage, isGitRepository, readme, indexHtmlProfile] = await Promise.all([
         measureDirectoryUsage(projectPath),
         isGitRepositoryDirectory(projectPath),
         readAgentProjectReadme(projectPath),
+        readAgentProjectIndexHtmlProfile(projectPath),
     ]);
-    const profile = resolveAgentProjectReadmeProfile({
+    const profile = resolveAgentProjectProfile({
         projectDirectoryName,
         readme,
+        indexHtmlTitle: indexHtmlProfile?.title ?? null,
+    });
+    const faviconRelativePath = await resolveAgentProjectFaviconRelativePath({
+        projectPath,
+        indexHtmlFaviconHref: indexHtmlProfile?.faviconHref ?? null,
     });
 
     return {
@@ -35,6 +43,7 @@ export async function createAgentProjectInfo(
         displayName: profile.displayName,
         description: profile.description,
         readmeFileName: profile.readmeFileName,
+        faviconRelativePath,
         relativePath: `${AGENT_PROJECTS_DIRECTORY_PATH}/${projectDirectoryName}`,
         absolutePath: projectPath,
         sizeBytes: projectUsage.sizeBytes,

package/apps/agents-server/src/utils/agentProjects/{resolveAgentProjectReadmeProfile.ts → resolveAgentProjectProfile.ts}

@@ -1,6 +1,7 @@
 import { parseMarkdownSection, splitMarkdownIntoSections } from '@promptbook-local/markdown-utils';
 import type { string_markdown } from '@promptbook-local/types';
 import { stripMarkdownText } from '../stripMarkdownText';
+import { humanizeAgentProjectName } from './humanizeAgentProjectName';
 import type { AgentProjectReadme } from './readAgentProjectReadme';
 
 /**
@@ -29,9 +30,9 @@ type AgentProjectReadmeSections = {
 };
 
 /**
- * Project profile values resolved from a project folder and optional README.
+ * Project profile values resolved from a project folder, its README and its `index.html`.
  */
-export type AgentProjectReadmeProfile = {
+export type AgentProjectProfile = {
     /**
      * User-facing project name.
      */
@@ -49,35 +50,30 @@ export type AgentProjectReadmeProfile = {
 };
 
 /**
- * Resolves display metadata for one agent project from its README.
+ * Resolves display metadata for one agent project.
  *
- * @param options - Project folder name and optional README content.
+ * @param options - Project folder name, optional README content and optional `index.html` title.
  * @returns Project display profile.
  */
-export function resolveAgentProjectReadmeProfile(options: {
+export function resolveAgentProjectProfile(options: {
     readonly projectDirectoryName: string;
     readonly readme: AgentProjectReadme | null;
-}): AgentProjectReadmeProfile {
-    const { projectDirectoryName, readme } = options;
-    if (!readme) {
-        return {
-            displayName: projectDirectoryName,
-            description: '',
-            readmeFileName: null,
-        };
-    }
-
-    const sections = parseMarkdownProfileSections(readme.content);
-    const displayName = resolveProjectDisplayName(
-        projectDirectoryName,
-        sections.firstExplicitHeadingSection?.title ?? null,
-    );
-    const description = resolveProjectDescription(readme.content, sections.firstSection?.content ?? readme.content);
+    readonly indexHtmlTitle: string | null;
+}): AgentProjectProfile {
+    const { projectDirectoryName, readme, indexHtmlTitle } = options;
+    const sections = readme === null ? null : parseMarkdownProfileSections(readme.content);
 
     return {
-        displayName,
-        description,
-        readmeFileName: readme.fileName,
+        displayName: resolveProjectDisplayName({
+            projectDirectoryName,
+            readmeHeadingTitle: sections?.firstExplicitHeadingSection?.title ?? null,
+            indexHtmlTitle,
+        }),
+        description:
+            readme === null
+                ? ''
+                : resolveProjectDescription(readme.content, sections?.firstSection?.content ?? readme.content),
+        readmeFileName: readme?.fileName ?? null,
     };
 }
 
@@ -112,19 +108,23 @@ function parseMarkdownProfileSections(markdown: string): AgentProjectReadmeSecti
 }
 
 /**
- * Resolves a project display name from the README heading or folder fallback.
+ * Resolves a project display name from the README heading, the `index.html` title or the folder name.
+ *
+ * A project which describes itself neither in a README nor in an `index.html` still gets a readable
+ * name, because its folder name is humanized — `prague-murders-map` reads as `Prague Murders Map`.
  *
- * @param projectDirectoryName - Project folder name.
- * @param parsedHeadingTitle - Title returned by the markdown section parser.
+ * @param options - Project folder name and the titles found in the project files.
  * @returns Display name.
  */
-function resolveProjectDisplayName(projectDirectoryName: string, parsedHeadingTitle: string | null): string {
-    if (!parsedHeadingTitle) {
-        return projectDirectoryName;
-    }
+function resolveProjectDisplayName(options: {
+    readonly projectDirectoryName: string;
+    readonly readmeHeadingTitle: string | null;
+    readonly indexHtmlTitle: string | null;
+}): string {
+    const readmeDisplayName = options.readmeHeadingTitle === null ? '' : stripMarkdownText(options.readmeHeadingTitle);
+    const indexHtmlDisplayName = (options.indexHtmlTitle || '').trim();
 
-    const displayName = stripMarkdownText(parsedHeadingTitle);
-    return displayName || projectDirectoryName;
+    return readmeDisplayName || indexHtmlDisplayName || humanizeAgentProjectName(options.projectDirectoryName);
 }
 
 /**

package/apps/agents-server/src/utils/agentProjects/resolveAgentProjectPublicUrls.ts

@@ -0,0 +1,34 @@
+import { listAgentProjectDomainRecords, resolveAgentProjectRuntimeBaseDomain } from './agentProjectRuntimeDomains';
+import { resolveCurrentAgentProjectServerDomain } from './resolveCurrentAgentProjectServerDomain';
+
+/**
+ * Resolves the stable public URLs of the projects of one agent.
+ *
+ * The public URL of one project comes from its persisted domain assignment, so it stays the same
+ * whether the project runtime is running or stopped.
+ *
+ * @param agentPermanentId - Permanent id of the agent owning the projects.
+ * @returns Public project URLs keyed by lowercase project name.
+ */
+export async function resolveAgentProjectPublicUrls(agentPermanentId: string): Promise<ReadonlyMap<string, string>> {
+    const [projectDomainRecords, currentServerDomain] = await Promise.all([
+        listAgentProjectDomainRecords(),
+        resolveCurrentAgentProjectServerDomain(),
+    ]);
+    const serverDomain = resolveAgentProjectRuntimeBaseDomain(currentServerDomain);
+    const publicUrlByProjectName = new Map<string, string>();
+
+    for (const projectDomainRecord of projectDomainRecords) {
+        if (projectDomainRecord.agentPermanentId.toLowerCase() !== agentPermanentId.toLowerCase()) {
+            continue;
+        }
+
+        if (serverDomain && resolveAgentProjectRuntimeBaseDomain(projectDomainRecord.serverDomain) !== serverDomain) {
+            continue;
+        }
+
+        publicUrlByProjectName.set(projectDomainRecord.projectName.toLowerCase(), projectDomainRecord.publicUrl);
+    }
+
+    return publicUrlByProjectName;
+}

package/apps/agents-server/src/utils/agentProjects/resolveAgentProjectRuntimeEnvironmentFlag.ts

@@ -0,0 +1,23 @@
+/**
+ * Values accepted as an enabled on/off environment flag.
+ */
+const ENABLED_ENVIRONMENT_FLAG_VALUES = ['1', 'true', 'yes', 'y'];
+
+/**
+ * Reads one optional on/off environment flag used by the agent project runtime managers.
+ *
+ * An unset (or empty) variable is reported as `null` instead of `false`, so every caller keeps
+ * ownership of its own default while the accepted spellings of "enabled" stay defined in one place.
+ *
+ * @param environmentVariableName - Name of the environment variable to read.
+ * @returns `true` or `false` when the variable is set, `null` when it is not.
+ */
+export function resolveAgentProjectRuntimeEnvironmentFlag(environmentVariableName: string): boolean | null {
+    const configuredValue = process.env[environmentVariableName]?.trim().toLowerCase();
+
+    if (!configuredValue) {
+        return null;
+    }
+
+    return ENABLED_ENVIRONMENT_FLAG_VALUES.includes(configuredValue);
+}

package/apps/agents-server/src/utils/agentProjects/resolveAgentProjectRuntimeStatus.ts

@@ -0,0 +1,73 @@
+import { formatAgentProjectRuntimeStatus } from './agentProjectRuntimeDisplay';
+import { resolveAgentProjectRuntime } from './agentProjectRuntimeRegistry';
+import { resolveAgentProjectPublicUrls } from './resolveAgentProjectPublicUrls';
+
+/**
+ * Current runtime state of one project as shown on its project page.
+ */
+export type AgentProjectRuntimeStatusInfo = {
+    /**
+     * Whether the project currently accepts requests on its assigned runtime port.
+     */
+    readonly isRunning: boolean;
+
+    /**
+     * Human-readable status label, the same one the project page shows.
+     */
+    readonly statusLabel: string;
+
+    /**
+     * Stable public project URL, or `null` when the project has no assigned domain or runtime.
+     *
+     * The URL stays the same while the project is stopped, so a link to it keeps working once the
+     * project is started again.
+     */
+    readonly projectUrl: string | null;
+};
+
+/**
+ * Resolves the runtime state of one single project.
+ *
+ * This is the one-project counterpart of `listAgentProjectChatReferences`, for surfaces which
+ * already know which project they show and must not pay for listing every project of the agent.
+ *
+ * @param agentPermanentId - Permanent id of the agent owning the project.
+ * @param projectName - Project directory name.
+ * @returns Runtime state of the project.
+ */
+export async function resolveAgentProjectRuntimeStatus(
+    agentPermanentId: string,
+    projectName: string,
+): Promise<AgentProjectRuntimeStatusInfo> {
+    const runtime = await resolveAgentProjectRuntime(agentPermanentId, projectName);
+
+    return {
+        isRunning: runtime?.isRunning ?? false,
+        statusLabel: formatAgentProjectRuntimeStatus(runtime),
+        projectUrl: runtime?.publicUrl ?? (await resolveAssignedAgentProjectPublicUrl(agentPermanentId, projectName)),
+    };
+}
+
+/**
+ * Resolves the domain assigned to one project which currently has no runtime.
+ *
+ * The assignment is scoped by the domain the current server is reached under, which is known from
+ * the request headers only. A caller outside a request — such as the local chat runner finishing a
+ * message — therefore learns no URL instead of failing, and the project page link still works.
+ *
+ * @param agentPermanentId - Permanent id of the agent owning the project.
+ * @param projectName - Project directory name.
+ * @returns Assigned public project URL, or `null` when there is none to resolve.
+ *
+ * @private helper of `resolveAgentProjectRuntimeStatus`
+ */
+async function resolveAssignedAgentProjectPublicUrl(
+    agentPermanentId: string,
+    projectName: string,
+): Promise<string | null> {
+    try {
+        return (await resolveAgentProjectPublicUrls(agentPermanentId)).get(projectName.toLowerCase()) ?? null;
+    } catch {
+        return null;
+    }
+}

package/apps/agents-server/src/utils/agentProjects/startDefaultAgentProjectRuntimes.ts

@@ -0,0 +1,110 @@
+import { createAgentProjectIdentityKey, type AgentProjectIdentity } from './agentProjectIdentity';
+import {
+    listAgentProjectRuntimeDesiredStates,
+    resolveAgentProjectRuntimeDesiredStateFromRecords,
+} from './agentProjectRuntimeDesiredState';
+import { listAgentProjectDomainRecords } from './agentProjectRuntimeDomains';
+import { resolveAgentProjectRuntime, startAgentProjectRuntime } from './agentProjectRuntimeRegistry';
+import { listAllLocalAgentProjectIdentities } from './listAllLocalAgentProjectIdentities';
+
+/**
+ * Outcome of one pass which brings all projects into their desired state.
+ */
+export type StartDefaultAgentProjectRuntimesReport = {
+    /**
+     * Number of projects started by this pass.
+     */
+    readonly startedProjectCount: number;
+
+    /**
+     * Number of projects which were already running or still coming up.
+     */
+    readonly runningProjectCount: number;
+
+    /**
+     * Number of projects left alone because they were explicitly stopped.
+     */
+    readonly stoppedProjectCount: number;
+
+    /**
+     * Number of projects which could not be started.
+     */
+    readonly failedProjectCount: number;
+};
+
+/**
+ * Starts every local project which is expected to be running but is not.
+ *
+ * This is what makes "running" the default state of a project: a project nobody explicitly stopped
+ * is started again after the server was restarted, after it crashed, and shortly after it was
+ * created. A project which somebody stopped stays stopped until it is started again.
+ *
+ * Projects are started one after another and one failing project never stops the pass, so a project
+ * with a broken dev command cannot keep every other project of the server down.
+ *
+ * @returns Summary of what the pass did.
+ */
+export async function startDefaultAgentProjectRuntimes(): Promise<StartDefaultAgentProjectRuntimesReport> {
+    const [projectIdentities, desiredStateRecords, projectDomainRecords] = await Promise.all([
+        listAllLocalAgentProjectIdentities(),
+        listAgentProjectRuntimeDesiredStates(),
+        listAgentProjectDomainRecords(),
+    ]);
+    const serverDomainByProjectKey = new Map(
+        projectDomainRecords.map((projectDomainRecord) => [
+            createAgentProjectIdentityKey(projectDomainRecord),
+            projectDomainRecord.serverDomain,
+        ]),
+    );
+
+    let startedProjectCount = 0;
+    let runningProjectCount = 0;
+    let stoppedProjectCount = 0;
+    let failedProjectCount = 0;
+
+    for (const projectIdentity of projectIdentities) {
+        if (resolveAgentProjectRuntimeDesiredStateFromRecords(desiredStateRecords, projectIdentity) === 'stopped') {
+            stoppedProjectCount++;
+            continue;
+        }
+
+        if (await isAgentProjectRuntimeAlreadyUp(projectIdentity)) {
+            runningProjectCount++;
+            continue;
+        }
+
+        try {
+            await startAgentProjectRuntime({
+                agentPermanentId: projectIdentity.agentPermanentId,
+                projectName: projectIdentity.projectName,
+                // Note: Keeps the project on the domain it was already assigned to, which is the only
+                //       way to tell servers of one VPS apart outside of a request.
+                serverDomain: serverDomainByProjectKey.get(createAgentProjectIdentityKey(projectIdentity)) ?? null,
+            });
+            startedProjectCount++;
+        } catch (error) {
+            failedProjectCount++;
+            console.error(
+                `[agent-project-runtimes] failed to start project \`${projectIdentity.projectName}\` of agent \`${projectIdentity.agentPermanentId}\``,
+                error,
+            );
+        }
+    }
+
+    return { startedProjectCount, runningProjectCount, stoppedProjectCount, failedProjectCount };
+}
+
+/**
+ * Returns whether one project already serves requests or is still coming up.
+ *
+ * A project which is still starting is left alone so that a slow dev command is not killed and
+ * restarted over and over by consecutive passes.
+ *
+ * @param projectIdentity - Agent and project identification.
+ * @returns `true` when the project must not be started again.
+ */
+async function isAgentProjectRuntimeAlreadyUp(projectIdentity: AgentProjectIdentity): Promise<boolean> {
+    const runtime = await resolveAgentProjectRuntime(projectIdentity.agentPermanentId, projectIdentity.projectName);
+
+    return Boolean(runtime && (runtime.isRunning || runtime.status === 'starting'));
+}

package/apps/agents-server/src/utils/agentReferenceResolver/MissingCoreAgentRecovery.ts

@@ -0,0 +1,19 @@
+/**
+ * Recovery offered for a missing agent reference that points to one of the bundled core agents.
+ *
+ * A core agent is never created from scratch like an ordinary missing reference — it is reinstated from the book
+ * bundled in the repository, together with every other core agent that happens to be missing at the same time.
+ *
+ * @private utility of Agents Server agent-reference diagnostics
+ */
+export type MissingCoreAgentRecovery = {
+    /**
+     * Titles of every bundled core agent currently missing on this server, including the referenced one.
+     */
+    readonly missingCoreAgentTitles: ReadonlyArray<string>;
+
+    /**
+     * Whether the current user is allowed to reinstate the missing core agents.
+     */
+    readonly isReinstateAllowed: boolean;
+};

package/apps/agents-server/src/utils/agentReferenceResolver/bookScopedAgentReferences.ts

@@ -2,6 +2,7 @@ import type { AgentReferenceResolver } from '../../../../../src/book-2.0/agent-s
 import { normalizeAgentName } from '../../../../../src/book-2.0/agent-source/normalizeAgentName';
 import { parseAgentSourceWithCommitments } from '../../../../../src/book-2.0/agent-source/parseAgentSourceWithCommitments';
 import { parseAgentSource } from '../../../../../src/book-2.0/agent-source/parseAgentSource';
+import { resolvePseudoAgentKindFromReference } from '../../../../../src/book-2.0/agent-source/pseudoAgentReferences';
 import type { string_book } from '../../../../../src/book-2.0/agent-source/string_book';
 import type { AgentCollection } from '../../../../../src/collection/agent-collection/AgentCollection';
 import type { BookCommitment } from '../../../../../src/commitments/_base/BookCommitment';
@@ -220,6 +221,12 @@ export function createBookScopedAgentReferenceResolver(options: {
                     chunks.push(content.slice(previousIndex, tokenMatch.index));
                     previousIndex = tokenMatch.index + tokenMatch.length;
 
+                    // Note: `Null`/`Void` (and other pseudo agents) are reserved language values, never embedded agents.
+                    if (resolvePseudoAgentKindFromReference(tokenReference)) {
+                        chunks.push(tokenMatch.token);
+                        continue;
+                    }
+
                     if (!tokenReference || !embeddedSources.has(normalizedTokenReference)) {
                         chunks.push(tokenMatch.token);
                         continue;