@memberjunction/server 6.2.0-edge.3 → 6.2.0-edge.5

package/src/resolvers/DatasetResolver.ts

@@ -1,6 +1,6 @@
 import { Arg, Ctx, Field, InputType, Int, ObjectType, Query, Resolver } from 'type-graphql';
-import { AppContext } from '../types.js';
-import { LogError, Metadata } from '@memberjunction/core';
+import { AppContext, UserPayload } from '../types.js';
+import { DatabaseProviderBase, DatasetItemFilterType, DatasetItemResultType, LogError, Metadata, UserInfo } from '@memberjunction/core';
 import { GetReadOnlyProvider } from '../util.js';
 import { ResolverBase } from '../generic/ResolverBase.js';
 
@@ -34,9 +34,64 @@ export class DatasetItemFilterTypeGQL {
   Filter: string;
 }
 
+/**
+ * Request handling shared by the dataset queries. Every read runs as the session's user, so the
+ * provider applies that user's entity read permission and row-level security per item, and every
+ * client ItemFilter passes the same AST screen as a RunView ExtraFilter before a provider sees it.
+ */
+export abstract class DatasetResolverBase extends ResolverBase {
+  /** The session's user. A dataset read never runs without one: the provider applies no read checks then. */
+  protected RequireDatasetUser(userPayload: UserPayload): UserInfo {
+    const user = this.GetUserFromPayload(userPayload);
+    if (!user) {
+      throw new Error('Dataset request refused: the session resolves to no user');
+    }
+    return user;
+  }
+
+  /**
+   * Screens every client ItemFilter: one statement, read-only, and subqueries only over entity base
+   * views the user can read. Throws on the first refusal. Returns the filters without null entries.
+   */
+  protected ScreenItemFilters(
+    itemFilters: DatasetItemFilterTypeGQL[] | undefined,
+    provider: DatabaseProviderBase,
+    user: UserInfo,
+  ): DatasetItemFilterType[] | undefined {
+    if (!itemFilters) return undefined;
+    itemFilters.forEach((itemFilter, index) => {
+      if (itemFilter) {
+        this.assertClientClauseUsesEntityBaseViews(itemFilter.Filter, `ItemFilters[${index}].Filter`, provider, user);
+      }
+    });
+    return itemFilters.filter((itemFilter) => itemFilter != null);
+  }
+
+  /**
+   * Applies the API encryption policy to each item's rows, as every other GraphQL read path does:
+   * an encrypted field the entity does not allow decrypted over the API leaves as ciphertext or the
+   * sentinel. Works on copies, because the rows can be the server cache's own frozen objects.
+   */
+  protected async ProtectEncryptedFields(
+    items: DatasetItemResultType[],
+    provider: DatabaseProviderBase,
+    user: UserInfo,
+  ): Promise<DatasetItemResultType[]> {
+    return Promise.all(
+      items.map(async (item) => {
+        const entity = provider.EntityByName(item.EntityName);
+        if (!item.Results?.length || !entity?.EncryptedFields.some((field) => !field.AllowDecryptInAPI)) {
+          return item;
+        }
+        const rows = item.Results.map((row: Record<string, unknown>) => ({ ...row }));
+        return { ...item, Results: await this.ArrayFilterEncryptedFieldsForAPI(item.EntityName, rows, user, provider, 'Name') };
+      }),
+    );
+  }
+}
 
 @Resolver(DatasetResultType)
-export class DatasetResolverExtended extends ResolverBase {
+export class DatasetResolverExtended extends DatasetResolverBase {
   @Query(() => DatasetResultType)
   async GetDatasetByName(
     @Arg('DatasetName', () => String) DatasetName: string,
@@ -45,10 +100,12 @@ export class DatasetResolverExtended extends ResolverBase {
   ) {
     // Check API key scope authorization for dataset read
     await this.CheckAPIKeyScopeAuthorization('dataset:read', DatasetName, userPayload);
+    const md = GetReadOnlyProvider(providers, {allowFallbackToReadWrite: true});
+    const user = this.RequireDatasetUser(userPayload);
+    const itemFilters = this.ScreenItemFilters(ItemFilters, md, user);
 
     try {
-      const md = GetReadOnlyProvider(providers, {allowFallbackToReadWrite: true});
-      const result = await md.GetDatasetByName(DatasetName, ItemFilters);
+      const result = await md.GetDatasetByName(DatasetName, itemFilters, user);
       if (result) {
         return {
           DatasetID: result.DatasetID,
@@ -56,7 +113,7 @@ export class DatasetResolverExtended extends ResolverBase {
           Success: result.Success,
           Status: result.Status,
           LatestUpdateDate: result.LatestUpdateDate,
-          Results: JSON.stringify(result.Results),
+          Results: JSON.stringify(await this.ProtectEncryptedFields(result.Results, md, user)),
         };
       } else {
         throw new Error('Error retrieving Dataset: ' + DatasetName);
@@ -90,7 +147,7 @@ export class DatasetStatusResultType {
 }
 
 @Resolver(DatasetStatusResultType)
-export class DatasetStatusResolver extends ResolverBase {
+export class DatasetStatusResolver extends DatasetResolverBase {
   @Query(() => DatasetStatusResultType)
   async GetDatasetStatusByName(
     @Arg('DatasetName', () => String) DatasetName: string,
@@ -99,10 +156,12 @@ export class DatasetStatusResolver extends ResolverBase {
   ) {
     // Check API key scope authorization for dataset read
     await this.CheckAPIKeyScopeAuthorization('dataset:read', DatasetName, userPayload);
+    const md = GetReadOnlyProvider(providers, {allowFallbackToReadWrite: true});
+    const user = this.RequireDatasetUser(userPayload);
+    const itemFilters = this.ScreenItemFilters(ItemFilters, md, user);
 
     try {
-      const md = GetReadOnlyProvider(providers, {allowFallbackToReadWrite: true});
-      const result = await md.GetDatasetStatusByName(DatasetName, ItemFilters);
+      const result = await md.GetDatasetStatusByName(DatasetName, itemFilters, user);
       if (result) {
         return {
           DatasetID: result.DatasetID,
@@ -131,12 +190,13 @@ export class DatasetStatusResolver extends ResolverBase {
     @Ctx() { providers, userPayload }: AppContext,
   ): Promise<DatasetStatusResultType[]> {
     const md = GetReadOnlyProvider(providers, {allowFallbackToReadWrite: true});
+    const user = this.RequireDatasetUser(userPayload);
     const results: DatasetStatusResultType[] = [];
 
     // Execute all status checks in parallel
     const statusPromises = DatasetNames.map(async (name) => {
       await this.CheckAPIKeyScopeAuthorization('dataset:read', name, userPayload);
-      return md.GetDatasetStatusByName(name);
+      return md.GetDatasetStatusByName(name, undefined, user);
     });
 
     const statuses = await Promise.all(statusPromises);

package/src/resolvers/EntityRecordNameResolver.ts

@@ -38,7 +38,8 @@ export class EntityRecordNameResult {
  *
  * Entity-level read permission is checked here ({@link ResolverBase.CheckUserReadPermissions}).
  * Field- and row-level security are applied by the provider's lookup, for the acting user passed
- * to it: when any field the name is built from is withheld, it answers with no name.
+ * to it: when any field the name is built from is withheld, it answers with no name, and it reads
+ * only rows the user's read row filter allows. Without an acting user there is no lookup.
  *
  * A withheld name answers `Success: false` with the same status as a record that does not exist.
  * Callers already treat "no name" as "show the primary key", and the identical answer keeps this
@@ -85,7 +86,7 @@ export class EntityRecordNameResolver extends ResolverBase {
     if (e) {
       const contextUser = userPayload ? this.GetUserFromPayload(userPayload) : undefined;
       // The acting user is what field- and row-level security are applied for.
-      const recordName = await md.GetEntityRecordName(e.Name, pk, contextUser);
+      const recordName = contextUser ? await md.GetEntityRecordName(e.Name, pk, contextUser) : '';
       if (recordName) return { Success: true, Status: 'OK', CompositeKey: pk, RecordName: recordName, EntityName };
       else
         return {

package/src/resolvers/FetchEntityVectorsResolver.ts

@@ -1,8 +1,9 @@
 import { Resolver, Query, Arg, Ctx, ObjectType, Field, Float, Int } from 'type-graphql';
 import { AppContext } from '../types.js';
-import { LogError, LogStatus, Metadata, RunView, UserInfo } from '@memberjunction/core';
+import { CompositeKey, EntityInfo, IMetadataProvider, LogError, LogStatus, Metadata, RunView, RunViewParams, UserInfo } from '@memberjunction/core';
 import { MJEntityDocumentEntity, MJVectorIndexEntity, MJVectorDatabaseEntity } from '@memberjunction/core-entities';
 import { ResolverBase } from '../generic/ResolverBase.js';
+import { GetReadOnlyProvider } from '../util.js';
 import { GetAIAPIKey } from '@memberjunction/ai';
 import { VectorDBBase } from '@memberjunction/ai-vectordb';
 import { AIEngine } from '@memberjunction/aiengine';
@@ -40,6 +41,15 @@ export class FetchEntityVectorsResult {
     ErrorMessage?: string;
 }
 
+/** A stored vector's listing row, as `fetchStoredVectors` reads it from `MJ: Entity Record Documents`. */
+type StoredVectorRow = { ID: string; RecordID: string };
+
+/** A stored vector chosen for the result, with the labels of the record it belongs to. */
+type LabelledVector = { row: StoredVectorRow; labels: Record<string, string> };
+
+/** Most Entity Record Document IDs named in one `ID IN (...)` filter when fetching VectorJSON. */
+const VECTOR_FETCH_CHUNK_SIZE = 1000;
+
 /* ───── Resolver ───── */
 
 @Resolver()
@@ -50,7 +60,7 @@ export class FetchEntityVectorsResolver extends ResolverBase {
         @Arg('entityDocumentID') entityDocumentID: string,
         @Arg('maxRecords', () => Int, { nullable: true }) maxRecords: number | undefined,
         @Arg('filter', { nullable: true }) filter: string | undefined,
-        @Ctx() { userPayload }: AppContext = {} as AppContext
+        @Ctx() { userPayload, providers }: AppContext = {} as AppContext
     ): Promise<FetchEntityVectorsResult> {
         const startTime = Date.now();
         try {
@@ -85,6 +95,17 @@ export class FetchEntityVectorsResolver extends ResolverBase {
                 );
             }
 
+            // Providers that keep their vectors in MJ, keyed by EntityDocumentID (the in-process
+            // SimpleVectorServiceProvider), can't be listed through QueryIndex: they key by the
+            // document ID rather than the index name, and return matches without vector values.
+            // Read the stored Entity Record Document rows instead.
+            if (vectorDBInstance.QueryKeyIsEntityDocumentID) {
+                const md = GetReadOnlyProvider(providers, { allowFallbackToReadWrite: true }) ?? Metadata.Provider;
+                const results = await this.fetchStoredVectors(entityDoc, limit, filter, currentUser, md);
+                LogStatus(`FetchEntityVectors: Loaded ${results.length} stored vectors for entity "${entityDoc.Entity}" in ${Date.now() - startTime}ms`);
+                return { Success: true, Results: results, TotalCount: results.length, ElapsedMs: Date.now() - startTime };
+            }
+
             // Step 4: Query with zero vector + Entity metadata filter.
             // Pinecone's list API doesn't support metadata filtering, but query does.
             // A zero vector returns results in arbitrary order (similarity is meaningless),
@@ -227,6 +248,162 @@ export class FetchEntityVectorsResolver extends ResolverBase {
         return instance;
     }
 
+    /**
+     * Load the vectors an EntityDocument stored in MJ (`MJ: Entity Record Documents.VectorJSON`),
+     * restricted to the entity records that pass `filter`. Each item's metadata carries the entity
+     * name, the record ID and the record's name fields, which is what the cluster view labels from.
+     */
+    private async fetchStoredVectors(
+        entityDoc: MJEntityDocumentEntity,
+        limit: number,
+        filter: string | undefined,
+        contextUser: UserInfo,
+        md: IMetadataProvider
+    ): Promise<EntityVectorItem[]> {
+        const entityInfo = md.EntityByName(entityDoc.Entity);
+        if (!entityInfo) {
+            throw new Error(`Entity "${entityDoc.Entity}" not found in metadata`);
+        }
+        const nameFields = this.labelFieldNames(entityInfo);
+        const { records, vectors } = await this.loadRecordsAndStoredVectors(entityInfo, entityDoc, nameFields, filter, contextUser);
+        const labelsByRecordID = this.mapRecordLabels(entityInfo, nameFields, records);
+
+        // VectorJSON is the heavy column: list without it, then fetch it only for the items returned.
+        const chosen = this.selectLabelledVectors(vectors, labelsByRecordID, limit);
+        const valuesByID = await this.loadVectorValues(chosen.map(c => c.row.ID), contextUser);
+
+        const results: EntityVectorItem[] = [];
+        for (const { row, labels } of chosen) {
+            const vectorJSON = valuesByID.get(row.ID);
+            if (!vectorJSON) continue; // removed or cleared since the listing
+            results.push({
+                ID: row.ID,
+                Values: JSON.parse(vectorJSON) as number[],
+                Metadata: JSON.stringify({ Entity: entityDoc.Entity, RecordID: row.RecordID, ...labels }),
+            });
+        }
+        return results;
+    }
+
+    /** The first `limit` stored vectors whose record was loaded, in listing order. */
+    private selectLabelledVectors(
+        vectors: StoredVectorRow[],
+        labelsByRecordID: Map<string, Record<string, string>>,
+        limit: number
+    ): LabelledVector[] {
+        const chosen: LabelledVector[] = [];
+        for (const row of vectors) {
+            if (chosen.length >= limit) break;
+            const labels = labelsByRecordID.get(row.RecordID.toLowerCase());
+            if (!labels) continue; // filtered out, not readable by this user, or the record no longer exists
+            chosen.push({ row, labels });
+        }
+        return chosen;
+    }
+
+    /**
+     * Fetch `VectorJSON` for the given Entity Record Document IDs, keyed by ID. The ID lists are
+     * capped at {@link VECTOR_FETCH_CHUNK_SIZE} per view and all sent in one `RunViews` batch.
+     */
+    private async loadVectorValues(ids: string[], contextUser: UserInfo): Promise<Map<string, string>> {
+        const valuesByID = new Map<string, string>();
+        if (ids.length === 0) return valuesByID;
+
+        const views: RunViewParams[] = [];
+        for (let i = 0; i < ids.length; i += VECTOR_FETCH_CHUNK_SIZE) {
+            const idList = ids.slice(i, i + VECTOR_FETCH_CHUNK_SIZE).map(id => `'${EscapeSQLString(id)}'`).join(',');
+            views.push({
+                EntityName: 'MJ: Entity Record Documents',
+                ExtraFilter: `ID IN (${idList})`,
+                Fields: ['ID', 'VectorJSON'],
+                IgnoreMaxRows: true,
+                BypassCache: true, // one-off ID lists: a cache slot per call would only hold memory
+                ResultType: 'simple',
+            });
+        }
+        const results = await new RunView().RunViews<{ ID: string; VectorJSON: string | null }>(views, contextUser);
+        for (const result of results) {
+            if (!result.Success) {
+                throw new Error(`Failed to load vector values: ${result.ErrorMessage}`);
+            }
+            for (const row of result.Results) {
+                if (row.VectorJSON) valuesByID.set(row.ID, row.VectorJSON);
+            }
+        }
+        return valuesByID;
+    }
+
+    /** The fields a stored vector is labelled with: the entity's name fields, else its NameField. */
+    private labelFieldNames(entityInfo: EntityInfo): string[] {
+        const nameFields = entityInfo.Fields.filter(f => f.IsNameField).map(f => f.Name);
+        if (nameFields.length === 0 && entityInfo.NameField) {
+            nameFields.push(entityInfo.NameField.Name);
+        }
+        return nameFields;
+    }
+
+    /**
+     * Load the records that pass `filter` (key and name fields only) and the list of the document's
+     * stored vectors (ID and RecordID, no VectorJSON). The two views don't depend on each other, so
+     * they go in one `RunViews` round trip.
+     */
+    private async loadRecordsAndStoredVectors(
+        entityInfo: EntityInfo,
+        entityDoc: MJEntityDocumentEntity,
+        nameFields: string[],
+        filter: string | undefined,
+        contextUser: UserInfo
+    ): Promise<{ records: Record<string, unknown>[]; vectors: StoredVectorRow[] }> {
+        const keyFields = entityInfo.PrimaryKeys.map(pk => pk.Name);
+        const [recordResult, docResult] = await new RunView().RunViews<Record<string, unknown>>([
+            {
+                EntityName: entityInfo.Name,
+                ExtraFilter: filter || undefined,
+                Fields: [...new Set([...keyFields, ...nameFields])],
+                IgnoreMaxRows: true,
+                ResultType: 'simple',
+            },
+            {
+                EntityName: 'MJ: Entity Record Documents',
+                ExtraFilter: `EntityDocumentID='${EscapeSQLString(entityDoc.ID)}' AND VectorJSON IS NOT NULL`,
+                Fields: ['ID', 'RecordID'],
+                IgnoreMaxRows: true,
+                // A cached RunView fetches every column so its slot can serve any field set, which
+                // would pull each vector's VectorJSON here. Bypass the cache so Fields is honoured.
+                BypassCache: true,
+                ResultType: 'simple',
+            },
+        ], contextUser);
+        if (!recordResult.Success) {
+            throw new Error(`Failed to load "${entityInfo.Name}" records: ${recordResult.ErrorMessage}`);
+        }
+        if (!docResult.Success) {
+            throw new Error(`Failed to load stored vectors: ${docResult.ErrorMessage}`);
+        }
+        return { records: recordResult.Results, vectors: docResult.Results as StoredVectorRow[] };
+    }
+
+    /**
+     * Map each loaded record to its name-field values, keyed by the same compact record-ID form
+     * the vector sync writes to `MJ: Entity Record Documents.RecordID` (lower-cased).
+     */
+    private mapRecordLabels(
+        entityInfo: EntityInfo,
+        nameFields: string[],
+        rows: Record<string, unknown>[]
+    ): Map<string, Record<string, string>> {
+        const map = new Map<string, Record<string, string>>();
+        for (const row of rows) {
+            const key = CompositeKey.FromEntityRecord(entityInfo, row).ToCompactURLSegment().toLowerCase();
+            const labels: Record<string, string> = {};
+            for (const name of nameFields) {
+                if (row[name] != null) labels[name] = String(row[name]);
+            }
+            map.set(key, labels);
+        }
+        return map;
+    }
+
     private errorResult(message: string, startTime: number): FetchEntityVectorsResult {
         return {
             Success: false,