@promptbook/cli 0.113.0-4 → 0.113.0-8

package/apps/agents-server/src/database/sqlite/localSqliteSupabase/localSqliteTableSchema.ts

@@ -0,0 +1,329 @@
+import type { LocalSqliteUpsertOptions } from './localSqliteTypes';
+import { uniqueStrings } from './localSqliteSql';
+
+/**
+ * Read index definition for hot SQLite queries.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteReadIndex = {
+    readonly name: string;
+    readonly columns: ReadonlyArray<string>;
+};
+
+/**
+ * Columns whose values are persisted as JSON text in local SQLite.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const JSON_COLUMNS_BY_TABLE = new Map<string, ReadonlySet<string>>([
+    ['Agent', new Set(['agentProfile', 'usage', 'preparedModelRequirements', 'preparedExternals'])],
+    ['AgentHistory', new Set([])],
+    ['ChatHistory', new Set(['message', 'usage'])],
+    ['LlmCache', new Set(['value'])],
+    ['VectorStoreKnowledgeSourceHashes', new Set([])],
+    ['Image', new Set([])],
+    ['File', new Set(['securityResult'])],
+    ['Message', new Set(['sender', 'recipients', 'metadata'])],
+    ['MessageSendAttempt', new Set(['raw'])],
+    ['UserChat', new Set(['messages'])],
+    ['UserChatJob', new Set(['parameters'])],
+    ['UserChatTimeout', new Set(['parameters'])],
+    ['UserData', new Set(['value'])],
+    ['Wallet', new Set(['jsonSchema'])],
+    ['ShareTargetPayload', new Set(['attachments'])],
+    ['CalendarConnection', new Set(['scopes'])],
+    ['CalendarActivity', new Set(['details'])],
+    ['ShibbolethUserIdentity', new Set(['rawAttributes'])],
+    ['ShibbolethAuthenticationAttempt', new Set(['rawAttributes'])],
+]);
+
+/**
+ * Columns that must always be surfaced as strings even when an older SQLite
+ * table was created with numeric affinity.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const TEXT_COLUMNS_BY_TABLE = new Map<string, ReadonlySet<string>>([['Metadata', new Set(['key', 'value', 'note'])]]);
+
+/**
+ * Table-specific integer columns whose names would otherwise be ambiguous.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const INTEGER_COLUMNS_BY_TABLE = new Map<string, ReadonlySet<string>>([['ServerLimit', new Set(['value'])]]);
+
+/**
+ * Boolean columns stored as integers by SQLite and restored as booleans.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const BOOLEAN_COLUMNS = new Set(['isAdmin', 'isRevoked', 'isGlobal', 'isUserScoped', 'isSuccessful', 'isChatFocused']);
+
+/**
+ * Tables whose primary key is provided as text rather than generated numerically.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const TEXT_PRIMARY_KEY_TABLES = new Set([
+    'UserChat',
+    'UserChatJob',
+    'UserChatTimeout',
+    'UserPushSubscription',
+    'ShareTargetPayload',
+]);
+
+/**
+ * Unique constraints required by common Supabase upsert and duplicate-detection flows.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const UNIQUE_INDEX_COLUMNS_BY_TABLE = new Map<string, ReadonlyArray<ReadonlyArray<string>>>([
+    ['_Server', [['name'], ['domain']]],
+    ['Metadata', [['key']]],
+    ['ServerLimit', [['key']]],
+    ['Agent', [['permanentId']]],
+    ['AgentExternals', [['type', 'hash']]],
+    ['VectorStoreKnowledgeSourceHashes', [['source']]],
+    ['User', [['username']]],
+    ['ShibbolethUserIdentity', [['userId'], ['email'], ['nameId']]],
+    ['UserChatJob', [['chatId', 'clientMessageId']]],
+    ['LlmCache', [['hash']]],
+    ['OpenAiAssistantCache', [['agentHash']]],
+    ['ApiTokens', [['token']]],
+    ['GenerationLock', [['lockKey']]],
+    ['CustomStylesheet', [['scope']]],
+    ['CustomJavascript', [['scope']]],
+    ['Wallet', [['userId', 'agentPermanentId', 'service', 'key']]],
+    ['UserData', [['userId', 'key']]],
+    ['UserPushSubscription', [['endpoint']]],
+]);
+
+/**
+ * Non-unique indexes for frequent standalone VPS reads.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const READ_INDEXES_BY_TABLE = new Map<string, ReadonlyArray<LocalSqliteReadIndex>>([
+    [
+        'Agent',
+        [
+            { name: 'agentName_lookup', columns: ['agentName'] },
+            { name: 'active_directory', columns: ['deletedAt', 'sortOrder', 'agentName'] },
+            { name: 'public_active_directory', columns: ['visibility', 'deletedAt', 'sortOrder', 'agentName'] },
+        ],
+    ],
+    [
+        'AgentFolder',
+        [{ name: 'active_directory', columns: ['deletedAt', 'parentId', 'sortOrder', 'name'] }],
+    ],
+    [
+        'UserChat',
+        [
+            { name: 'user_agent_source_createdAt', columns: ['userId', 'agentPermanentId', 'source', 'createdAt'] },
+            { name: 'agent_source_user_createdAt', columns: ['agentPermanentId', 'source', 'userId', 'createdAt'] },
+        ],
+    ],
+    [
+        'UserChatJob',
+        [
+            { name: 'ready_queue', columns: ['status', 'cancelRequestedAt', 'queuedAt', 'createdAt'] },
+            { name: 'active_chat_scope', columns: ['chatId', 'userId', 'agentPermanentId', 'status', 'createdAt'] },
+            { name: 'agent_chat_status', columns: ['agentPermanentId', 'chatId', 'status'] },
+            { name: 'running_lease', columns: ['status', 'leaseExpiresAt'] },
+        ],
+    ],
+    [
+        'UserChatTimeout',
+        [
+            { name: 'ready_due', columns: ['status', 'cancelRequestedAt', 'pausedAt', 'dueAt', 'createdAt'] },
+            {
+                name: 'active_chat_scope',
+                columns: ['chatId', 'userId', 'agentPermanentId', 'status', 'pausedAt', 'dueAt', 'createdAt'],
+            },
+            { name: 'running_lease', columns: ['status', 'leaseExpiresAt'] },
+        ],
+    ],
+]);
+
+/**
+ * Known unique conflict columns used when `.upsert` omits `onConflict`.
+ *
+ * @private constant of `$provideLocalSqliteSupabase`
+ */
+const DEFAULT_UPSERT_CONFLICT_COLUMNS_BY_TABLE = new Map<string, ReadonlyArray<string>>([
+    ['AgentExternals', ['type', 'hash']],
+    ['LlmCache', ['hash']],
+    ['VectorStoreKnowledgeSourceHashes', ['source']],
+    ['Metadata', ['key']],
+    ['ServerLimit', ['key']],
+]);
+
+/**
+ * Resolves a table base name from the actual prefixed table name.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveTableBaseName(tableName: string): string {
+    const knownTableNames = uniqueStrings([
+        '_Server',
+        ...Array.from(JSON_COLUMNS_BY_TABLE.keys()),
+        ...Array.from(UNIQUE_INDEX_COLUMNS_BY_TABLE.keys()),
+        ...Array.from(TEXT_PRIMARY_KEY_TABLES),
+        'AgentFolder',
+        'CustomStylesheet',
+        'CustomJavascript',
+        'CalendarActivity',
+    ]).sort((left, right) => right.length - left.length);
+
+    return knownTableNames.find((knownTableName) => tableName.endsWith(knownTableName)) || tableName;
+}
+
+/**
+ * Resolves columns participating in known unique indexes.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveUniqueIndexColumns(tableBaseName: string): Array<string> {
+    return resolveUniqueIndexes(tableBaseName).flatMap((columns) => [...columns]);
+}
+
+/**
+ * Resolves columns participating in known read indexes.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveReadIndexColumns(tableBaseName: string): Array<string> {
+    return resolveReadIndexes(tableBaseName).flatMap((readIndex) => [...readIndex.columns]);
+}
+
+/**
+ * Resolves upsert conflict columns.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveUpsertConflictColumns(
+    tableBaseName: string,
+    options: LocalSqliteUpsertOptions,
+): ReadonlyArray<string> {
+    if (options.onConflict) {
+        return options.onConflict
+            .split(',')
+            .map((column) => column.trim())
+            .filter(Boolean);
+    }
+
+    return DEFAULT_UPSERT_CONFLICT_COLUMNS_BY_TABLE.get(tableBaseName) || [];
+}
+
+/**
+ * Resolves the unique index definitions declared for a table base name.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveUniqueIndexes(tableBaseName: string): ReadonlyArray<ReadonlyArray<string>> {
+    return UNIQUE_INDEX_COLUMNS_BY_TABLE.get(tableBaseName) || [];
+}
+
+/**
+ * Resolves the read index definitions declared for a table base name.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveReadIndexes(tableBaseName: string): ReadonlyArray<LocalSqliteReadIndex> {
+    return READ_INDEXES_BY_TABLE.get(tableBaseName) || [];
+}
+
+/**
+ * Resolves whether a column is JSON for a specific table.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function isJsonColumn(tableName: string, column: string): boolean {
+    return isJsonColumnForTableBaseName(resolveTableBaseName(tableName), column);
+}
+
+/**
+ * Resolves whether a column should be forced to a string for a specific table.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function isTextColumn(tableName: string, column: string): boolean {
+    return isTextColumnForTableBaseName(resolveTableBaseName(tableName), column);
+}
+
+/**
+ * Resolves whether a column is a boolean stored as an integer.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function isBooleanColumn(column: string): boolean {
+    return BOOLEAN_COLUMNS.has(column);
+}
+
+/**
+ * Resolves whether a table base name uses a text primary key.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function isTextPrimaryKeyTable(tableBaseName: string): boolean {
+    return TEXT_PRIMARY_KEY_TABLES.has(tableBaseName);
+}
+
+/**
+ * Resolves SQLite column affinity for dynamically added columns.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function resolveSqliteColumnType(tableBaseName: string, column: string): string {
+    if (isTextColumnForTableBaseName(tableBaseName, column) || isJsonColumnForTableBaseName(tableBaseName, column)) {
+        return 'TEXT';
+    }
+    if (BOOLEAN_COLUMNS.has(column)) {
+        return 'INTEGER';
+    }
+    if (isIntegerColumnForTableBaseName(tableBaseName, column)) {
+        return 'INTEGER';
+    }
+    if (
+        column === 'id' ||
+        column.endsWith('Id') ||
+        column.endsWith('Count') ||
+        column.endsWith('Ms') ||
+        column.endsWith('Bytes') ||
+        column === 'sortOrder' ||
+        column === 'attemptCount' ||
+        column === 'runCount'
+    ) {
+        return 'INTEGER';
+    }
+
+    return 'TEXT';
+}
+
+/**
+ * Resolves whether a column is JSON for a specific table base name.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+function isJsonColumnForTableBaseName(tableBaseName: string, column: string): boolean {
+    return JSON_COLUMNS_BY_TABLE.get(tableBaseName)?.has(column) || false;
+}
+
+/**
+ * Resolves whether a column should be forced to a string for a specific table base name.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+function isTextColumnForTableBaseName(tableBaseName: string, column: string): boolean {
+    return TEXT_COLUMNS_BY_TABLE.get(tableBaseName)?.has(column) || false;
+}
+
+/**
+ * Resolves whether a column is known to be numeric for a specific table base name.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+function isIntegerColumnForTableBaseName(tableBaseName: string, column: string): boolean {
+    return INTEGER_COLUMNS_BY_TABLE.get(tableBaseName)?.has(column) || false;
+}

package/apps/agents-server/src/database/sqlite/localSqliteSupabase/localSqliteTypes.ts

@@ -0,0 +1,84 @@
+import type { TODO_any } from '@promptbook-local/types';
+
+/**
+ * Minimal query result shape consumed by Agents Server Supabase call sites.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteQueryResult<TData = TODO_any> = {
+    readonly data: TData | null;
+    readonly error: LocalSqliteError | null;
+    readonly count?: number | null;
+    readonly status?: number;
+    readonly statusText?: string;
+};
+
+/**
+ * Supabase-like error shape returned by the local SQLite adapter.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteError = {
+    readonly code?: string;
+    readonly message: string;
+    readonly details?: string;
+    readonly hint?: string;
+};
+
+/**
+ * Supported query operation kinds.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteOperation = 'select' | 'insert' | 'update' | 'delete' | 'upsert';
+
+/**
+ * Query filter captured from Supabase-like fluent calls.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteFilter = {
+    readonly column: string;
+    readonly operator: 'eq' | 'neq' | 'is' | 'not-is' | 'in' | 'lt' | 'lte' | 'gt' | 'gte' | 'like' | 'ilike';
+    readonly value: unknown;
+};
+
+/**
+ * Query order captured from Supabase-like fluent calls.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteOrder = {
+    readonly column: string;
+    readonly ascending: boolean;
+    readonly nullsFirst?: boolean;
+};
+
+/**
+ * Select options supported by Supabase and used by this app.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteSelectOptions = {
+    readonly count?: 'exact';
+    readonly head?: boolean;
+};
+
+/**
+ * Upsert options supported by Supabase and used by this app.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteUpsertOptions = {
+    readonly onConflict?: string;
+};
+
+/**
+ * SQL clause fragment paired with its positional bind values.
+ *
+ * @private type of `$provideLocalSqliteSupabase`
+ */
+export type LocalSqliteSqlFragment = {
+    readonly sql: string;
+    readonly values: ReadonlyArray<unknown>;
+};

package/apps/agents-server/src/database/sqlite/localSqliteSupabase/localSqliteValueCodec.ts

@@ -0,0 +1,63 @@
+import { isBooleanColumn, isJsonColumn, isTextColumn } from './localSqliteTableSchema';
+
+/**
+ * Serializes one value for SQLite storage.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function serializeValue(tableName: string, column: string, value: unknown): unknown {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (value === null) {
+        return null;
+    }
+    if (isTextColumn(tableName, column)) {
+        return String(value);
+    }
+    if (isJsonColumn(tableName, column)) {
+        return typeof value === 'string' ? value : JSON.stringify(value);
+    }
+    if (isBooleanColumn(column)) {
+        return value ? 1 : 0;
+    }
+    return value;
+}
+
+/**
+ * Deserializes one SQLite row into Supabase-like row values.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function deserializeRow(tableName: string, row: Record<string, unknown>): Record<string, unknown> {
+    const result: Record<string, unknown> = {};
+
+    for (const [column, value] of Object.entries(row)) {
+        if (value === null || value === undefined) {
+            result[column] = null;
+        } else if (isTextColumn(tableName, column)) {
+            result[column] = String(value);
+        } else if (isJsonColumn(tableName, column) && typeof value === 'string') {
+            result[column] = parseJsonValue(value);
+        } else if (isBooleanColumn(column)) {
+            result[column] = Boolean(value);
+        } else {
+            result[column] = value;
+        }
+    }
+
+    return result;
+}
+
+/**
+ * Parses JSON while preserving invalid strings.
+ *
+ * @private function of `deserializeRow`
+ */
+function parseJsonValue(value: string): unknown {
+    try {
+        return JSON.parse(value);
+    } catch {
+        return value;
+    }
+}

package/apps/agents-server/src/database/sqlite/localSqliteSupabase/normalizeSqliteError.ts

@@ -0,0 +1,22 @@
+import type { LocalSqliteError } from './localSqliteTypes';
+
+/**
+ * Converts SQLite errors into Supabase-like errors.
+ *
+ * @private function of `$provideLocalSqliteSupabase`
+ */
+export function normalizeSqliteError(error: unknown): LocalSqliteError {
+    const message = error instanceof Error ? error.message : String(error);
+    const sqliteCode =
+        typeof error === 'object' && error !== null && typeof (error as { code?: unknown }).code === 'string'
+            ? (error as { code: string }).code
+            : undefined;
+
+    return {
+        code:
+            sqliteCode === 'SQLITE_CONSTRAINT_UNIQUE' || sqliteCode === 'SQLITE_CONSTRAINT_PRIMARYKEY'
+                ? '23505'
+                : sqliteCode,
+        message,
+    };
+}

package/apps/agents-server/src/search/createDefaultServerSearchProviders/loadLocalOrganizationSearchDataset.ts

@@ -2,7 +2,9 @@ import { $getTableName } from '../../database/$getTableName';
 import { $provideSupabaseForServer } from '../../database/$provideSupabaseForServer';
 import type { AgentsServerDatabase } from '../../database/schema';
 import type { AgentBasicInformation } from '../../../../../src/book-2.0/agent-source/AgentBasicInformation';
+import { $provideAgentCollectionForServer } from '../../tools/$provideAgentCollectionForServer';
 import { $provideAgentReferenceResolver } from '../../utils/agentReferenceResolver/$provideAgentReferenceResolver';
+import { createLocalAgentSourceImporter } from '../../utils/createLocalAgentSourceImporter';
 import { getWellKnownAgentUrl } from '../../utils/getWellKnownAgentUrl';
 import { resolveCurrentOrInternalServerOrigin } from '../../utils/resolveCurrentOrInternalServerOrigin';
 import { resolveStoredAgentStates } from '../../utils/resolveStoredAgentState';
@@ -85,12 +87,21 @@ export async function loadLocalOrganizationSearchDataset(options: {
     const folders = (folderResult.data || []) as AgentFolderSearchRow[];
     const localServerUrl = await resolveCurrentOrInternalServerOrigin();
     const agentReferenceResolver = await $provideAgentReferenceResolver();
+    const adamAgentUrl = await getWellKnownAgentUrl('ADAM');
+    const agentSourceImporter = createLocalAgentSourceImporter({
+        collection: await $provideAgentCollectionForServer(),
+        localServerUrls: [localServerUrl],
+        localAgentUrls: [adamAgentUrl],
+        adamAgentUrl,
+        fallbackResolver: agentReferenceResolver,
+    });
     const resolvedAgents = await resolveStoredAgentStates(
         (agentResult.data || []) as Array<Pick<AgentSearchRow, 'id' | 'agentName' | 'permanentId' | 'agentSource' | 'folderId' | 'visibility'>>,
         {
             localServerUrl,
-            adamAgentUrl: await getWellKnownAgentUrl('ADAM'),
+            adamAgentUrl,
             agentReferenceResolver,
+            agentSourceImporter,
         },
     );
     const agents = resolvedAgents.map((agent) => ({

package/apps/agents-server/src/tools/send_email.ts

@@ -4,7 +4,6 @@ import type { Message } from '../../../../src/types/Message';
 import type { OutboundEmail } from '../message-providers/email/_common/Email';
 import { parseEmailAddresses } from '../message-providers/email/_common/utils/parseEmailAddresses';
 import { SmtpMessageProvider } from '../message-providers/email/smtp/SmtpMessageProvider';
-import { humanizeOutboundEmail } from '../utils/messages/humanizeOutboundEmail';
 import { parseUseEmailSmtpCredential } from '../utils/messages/parseUseEmailSmtpCredential';
 import { isSendMessageDeliveryError, sendMessage, type SendMessageResult } from '../utils/messages/sendMessage';
 import {
@@ -82,7 +81,7 @@ export async function send_email(args: SendEmailToolArgs): Promise<string> {
     }
 
     const smtpCredential = parseUseEmailSmtpCredential(smtpCredentialRaw);
-    const email = humanizeOutboundEmail(normalizeSendEmailPayload(args, smtpCredential.fromAddress || defaultFromAddress));
+    const email = normalizeSendEmailPayload(args, smtpCredential.fromAddress || defaultFromAddress);
     const smtpProvider = new SmtpMessageProvider({
         host: smtpCredential.host,
         port: smtpCredential.port,

package/apps/agents-server/src/utils/agentOrganization/loadAgentOrganizationState.ts

@@ -2,7 +2,9 @@ import type { AgentBasicInformation } from '../../../../../src/book-2.0/agent-so
 import { $getTableName } from '../../database/$getTableName';
 import { $provideSupabaseForServer } from '../../database/$provideSupabaseForServer';
 import type { AgentsServerDatabase } from '../../database/schema';
+import { $provideAgentCollectionForServer } from '../../tools/$provideAgentCollectionForServer';
 import { $provideAgentReferenceResolver } from '../agentReferenceResolver/$provideAgentReferenceResolver';
+import { createLocalAgentSourceImporter } from '../createLocalAgentSourceImporter';
 import { getCurrentUser } from '../getCurrentUser';
 import { getWellKnownAgentUrl } from '../getWellKnownAgentUrl';
 import { resolveCurrentOrInternalServerOrigin } from '../resolveCurrentOrInternalServerOrigin';
@@ -187,14 +189,24 @@ async function loadResolvedOrganizationSnapshot(options: {
 
     const localServerUrl = await resolveCurrentOrInternalServerOrigin();
     const agentReferenceResolver = await $provideAgentReferenceResolver();
+    const adamAgentUrl = await getWellKnownAgentUrl('ADAM');
+    const collection = await $provideAgentCollectionForServer();
+    const agentSourceImporter = createLocalAgentSourceImporter({
+        collection,
+        localServerUrls: [localServerUrl],
+        localAgentUrls: [adamAgentUrl],
+        adamAgentUrl,
+        fallbackResolver: agentReferenceResolver,
+    });
     const resolvedAgents = await resolveStoredAgentStates(
         (agentResult.data || []) as Array<
             Pick<AgentRow, 'agentName' | 'agentSource' | 'permanentId' | 'visibility' | 'folderId' | 'sortOrder'>
         >,
         {
             localServerUrl,
-            adamAgentUrl: await getWellKnownAgentUrl('ADAM'),
+            adamAgentUrl,
             agentReferenceResolver,
+            agentSourceImporter,
         },
     );
     const agents = resolvedAgents.map((agent) =>