lucid-extension-sdk 0.0.149 → 0.0.152

package/core/data/datasource/datasourceutils.d.ts

@@ -7,7 +7,7 @@ export declare function alphabetize(n: number): string;
 /** @ignore */
 export declare const MetadataPK = "__PK__";
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
-export declare function makeSerializedImportedCollection(id: string, name: string, rawSchema: string[], data: SerializedFields[], metadata?: {
+export declare function makeSerializedImportedCollection(id: string, hash: string, name: string, rawSchema: string[], data: SerializedFields[], metadata?: {
     [key: string]: SerializedFields[];
 }): SerializedImportedCollection;
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */

package/core/data/datasource/datasourceutils.js

@@ -99,7 +99,7 @@ function makeSerializedItems(rawItems, oldToNewFields, isMetadata = false) {
     return serializedItems;
 }
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
-function makeSerializedImportedCollection(id, name, rawSchema, data, metadata) {
+function makeSerializedImportedCollection(id, hash, name, rawSchema, data, metadata) {
     const { sheetSchema, oldToNewFields } = makeSchema(rawSchema);
     const metadataCollections = {};
     const itemOrderCollection = makeItemOrderCollection(data.length, name);
@@ -120,6 +120,7 @@ function makeSerializedImportedCollection(id, name, rawSchema, data, metadata) {
             'properties': {
                 'UpstreamType': upstreamupdatetype_1.UpstreamUpdateType.EVENTPULL,
                 'sheetId': id,
+                'hash': hash,
             },
         },
         'Metadata': metadataCollections,

package/core/data/datasource/serializedimporteddatasource.d.ts

@@ -17,7 +17,6 @@ export declare const isSerializedImportedMetadataCollection: (subject: unknown)
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
 export interface SerializedImportedCollection {
     'Name': string;
-    'Id'?: string;
     'Schema': SerializedSchema;
     'Items': SerializedDataItems;
     'UpstreamConfig': {

package/core/jsonserializable.d.ts

@@ -13,3 +13,5 @@ export interface JsonObjectConvertible {
  * Any type that can be natively converted to a string with JSON.stringify.
  */
 export declare type JsonSerializable = undefined | null | string | number | boolean | JsonArray | JsonObject;
+export declare function isJsonSerializable(x: unknown): x is JsonSerializable;
+export declare function isJsonObject(x: unknown): x is JsonObject;

package/core/jsonserializable.js

@@ -1,2 +1,25 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.isJsonObject = exports.isJsonSerializable = void 0;
+const checks_1 = require("./checks");
+function isJsonSerializable(x) {
+    return (x == null ||
+        (0, checks_1.isString)(x) ||
+        (0, checks_1.isNumber)(x) ||
+        (0, checks_1.isBoolean)(x) ||
+        ((0, checks_1.isArray)(x) && x.every(isJsonSerializable)) ||
+        isJsonObject(x));
+}
+exports.isJsonSerializable = isJsonSerializable;
+function isJsonObject(x) {
+    if (!(0, checks_1.isObject)(x)) {
+        return false;
+    }
+    for (const k in x) {
+        if (!isJsonSerializable(x[k])) {
+            return false;
+        }
+    }
+    return true;
+}
+exports.isJsonObject = isJsonObject;

package/core/result.d.ts

@@ -0,0 +1,11 @@
+export interface Success<T> {
+    value: T;
+}
+export interface Failure<T> {
+    error: T;
+}
+export declare type Result<S, F> = Success<S> | Failure<F>;
+export declare function isSuccess<S, F>(res: Result<S, F>): res is Success<S>;
+export declare function isFailure<S, F>(res: Result<S, F>): res is Failure<F>;
+export declare function groupByResultType<S, F>(results: Result<S, F>[]): [Success<S>[], Failure<F>[]];
+export declare function isEqual<S, F>(a: Result<S, F> | undefined, b: Result<S, F> | undefined, valueCompS: (p1: S, p2: S) => boolean, valueCompF: (p1: F, p2: F) => boolean): boolean;

package/core/result.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isEqual = exports.groupByResultType = exports.isFailure = exports.isSuccess = void 0;
+const checks_1 = require("./checks");
+function isSuccess(res) {
+    return (0, checks_1.isDef)(res.value);
+}
+exports.isSuccess = isSuccess;
+function isFailure(res) {
+    return (0, checks_1.isDef)(res.error);
+}
+exports.isFailure = isFailure;
+function groupByResultType(results) {
+    const successes = [];
+    const failures = [];
+    results.forEach((result) => {
+        if (isSuccess(result)) {
+            successes.push(result);
+        }
+        else {
+            failures.push(result);
+        }
+    });
+    return [successes, failures];
+}
+exports.groupByResultType = groupByResultType;
+function isEqual(a, b, valueCompS, valueCompF) {
+    return ((!!a &&
+        !!b &&
+        ((isSuccess(a) && isSuccess(b) && valueCompS(a.value, b.value)) ||
+            (isFailure(a) && isFailure(b) && valueCompF(a.error, b.error)))) ||
+        a == b);
+}
+exports.isEqual = isEqual;

package/core/spreadsheetintegration/lucidspreadsheetintegration.d.ts

@@ -19,9 +19,10 @@ export declare type ImportedResults = {
         'collectionId': string;
         'headerRow': number;
         'schema': SerializedSchema;
-        'upstreamConfig': JsonSerializable;
+        'upstreamConfig'?: JsonSerializable;
     }[];
 };
+export declare const isImportedResults: (x: unknown) => x is ImportedResults;
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
 export declare abstract class LucidSpreadsheetIntegration<CONFIG extends JsonSerializable, SHEET extends JsonSerializable> {
     abstract labelDescription: string;
@@ -30,5 +31,5 @@ export declare abstract class LucidSpreadsheetIntegration<CONFIG extends JsonSer
     abstract getSpreadsheetDetailsToImport(): Promise<CONFIG | LucidSpreadsheetIntegrationFailureType>;
     abstract getMultipleSheetsForSpreadsheetDetails(spreadsheetDetails: CONFIG): Promise<Map<SHEET, string> | LucidSpreadsheetIntegrationFailureType>;
     abstract getSpreadsheetData(spreadsheetDetails: CONFIG, sheets: SHEET[]): Promise<SerializedImportedDataSource | LucidSpreadsheetIntegrationFailureType>;
-    abstract importIsComplete(importedResults: ImportedResults): void;
+    abstract importIsComplete(syncDataSourceIdNonce: string, importedResults: ImportedResults): void;
 }

package/core/spreadsheetintegration/lucidspreadsheetintegration.js

@@ -1,6 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LucidSpreadsheetIntegration = exports.lucidSpreadsheetIntegrationFailureTypeValidator = exports.LucidSpreadsheetIntegrationFailureType = void 0;
+exports.LucidSpreadsheetIntegration = exports.isImportedResults = exports.lucidSpreadsheetIntegrationFailureTypeValidator = exports.LucidSpreadsheetIntegrationFailureType = void 0;
+const checks_1 = require("../checks");
+const serializedupstreamconfig_1 = require("../data/datasource/serializedupstreamconfig");
+const serializedschema_1 = require("../data/serializedfield/serializedschema");
+const jsonserializable_1 = require("../jsonserializable");
 const validators_1 = require("../validators/validators");
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
 var LucidSpreadsheetIntegrationFailureType;
@@ -11,6 +15,17 @@ var LucidSpreadsheetIntegrationFailureType;
 })(LucidSpreadsheetIntegrationFailureType = exports.LucidSpreadsheetIntegrationFailureType || (exports.LucidSpreadsheetIntegrationFailureType = {}));
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
 exports.lucidSpreadsheetIntegrationFailureTypeValidator = (0, validators_1.stringEnumValidator)(LucidSpreadsheetIntegrationFailureType);
+exports.isImportedResults = (0, validators_1.strictObjectValidator)({
+    'dataSourceId': checks_1.isString,
+    'syncDataSourceId': checks_1.isString,
+    'upstreamConfig': serializedupstreamconfig_1.isSerializedUpstreamConfig,
+    'collections': (0, validators_1.arrayValidator)((0, validators_1.strictObjectValidator)({
+        'collectionId': checks_1.isString,
+        'headerRow': checks_1.isNumber,
+        'schema': serializedschema_1.isSerializedSchema,
+        'upstreamConfig': jsonserializable_1.isJsonSerializable,
+    })),
+});
 /** @ignore until spreadsheet integration is ready for launch (CHART-51946) */
 class LucidSpreadsheetIntegration {
 }

package/core/spreadsheetintegration/lucidspreadsheetintegrationregistry.js

@@ -44,7 +44,7 @@ class LucidSpreadsheetIntegrationRegistry {
         });
         const importIsCompleteActionName = LucidSpreadsheetIntegrationRegistry.nextHookName();
         client.registerAction(importIsCompleteActionName, async (message) => {
-            spreadsheetIntegration.importIsComplete(message['ImportedResults']);
+            spreadsheetIntegration.importIsComplete(message['SyncDataSourceIdNonce'], message['ImportedResults']);
         });
         const serialized = {
             'ld': spreadsheetIntegration.labelDescription,

package/dataconnector/datasourceclient.d.ts

@@ -17,8 +17,22 @@ export declare class DataSourceClient {
      * and items. If it's just an update then the schema can be omitted (if unchanged) and items that already exist can
      * be partial.*/
     update(request: DataSourceRequest): Promise<void>;
+    /**
+     * When dealing with data, we can often have a very large number of collections in a single data source, and in
+     * turn, we may well end up get the values for a very large number of metadata values per collection.
+     * Because the endpoints for metadata use query params to encode the metadata names, and because metadata value
+     * reads can be easily parallelized, and because URLs have length limits, this function constructs an array of
+     * query params that are certain to be short enough that each individual query will be a safe length, while still
+     * getting *all* of the requested data.
+     * @param keys The metadata keys we are requesting from the data sync service for the corresponding data source.
+     * @param prefixes The metadata prefixes we are requesting from the data sync service for the corresponding data source.
+     * @returns An array of query params such that each query param is short enough to be safe, but all params combined
+     *             cover all requested metadata.
+     */
+    private getLengthLimitedQueryParams;
+    private getResponseOrError;
     /** @ignore until metadata endpoints are made part of the public API */
-    getMetadata(keys: string[]): Promise<MetadataRecord[]>;
+    getMetadata(keys: string[], prefixes?: string[]): Promise<MetadataRecord[]>;
     /** @ignore until metadata endpoints are made part of the public API */
     patchMetadata(patches: MetadataPatch[]): Promise<MetadataPatchResponse>;
 }
@@ -30,14 +44,14 @@ export declare class MockDataSourceClient extends DataSourceClient {
     /** Assign this to your mocked update function */
     gotUpdate: (request: DataSourceRequest) => void;
     /** Assign this to your mocked update function */
-    gotMetadata: (keys: string[]) => MetadataRecord[];
+    gotMetadata: (keys: string[], prefixes: string[]) => MetadataRecord[];
     /** Assign this to your mocked update function */
     patchedMetadata: (patches: MetadataPatch[]) => MetadataPatchResponse;
     constructor();
     /** @ignore */
     update(request: DataSourceRequest): Promise<void>;
     /** @ignore */
-    getMetadata(keys: string[]): Promise<MetadataRecord[]>;
+    getMetadata(keys: string[], prefixes?: string[]): Promise<MetadataRecord[]>;
     /** @ignore */
     patchMetadata(patches: MetadataPatch[]): Promise<MetadataPatchResponse>;
 }

package/dataconnector/datasourceclient.js

@@ -2,12 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MockDataSourceClient = exports.DataSourceClient = void 0;
 const checks_1 = require("../core/checks");
+const result_1 = require("../core/result");
 const validators_1 = require("../core/validators/validators");
 const dataconnector_1 = require("./dataconnector");
 const datasourcemetadatatypes_1 = require("./datasourcemetadatatypes");
 const datasourceupdatetypes_1 = require("./datasourceupdatetypes");
 const defaultfetchfunction_1 = require("./defaultfetchfunction");
 const throwunsuccessful_1 = require("./throwunsuccessful");
+const QueryParamLengthLimit = 2000;
 /**
  * Authenticated client for DataSource related requests to Lucid.
  */
@@ -61,14 +63,42 @@ class DataSourceClient {
         // return await response.text();
         // but lets avoid users trying to interpret the response
     }
-    /** @ignore until metadata endpoints are made part of the public API */
-    async getMetadata(keys) {
-        if (this.metadataUrl == null) {
-            throw new Error('Attempted to access data sync service metadata endpoints in environment where the URL does not exist!');
+    /**
+     * When dealing with data, we can often have a very large number of collections in a single data source, and in
+     * turn, we may well end up get the values for a very large number of metadata values per collection.
+     * Because the endpoints for metadata use query params to encode the metadata names, and because metadata value
+     * reads can be easily parallelized, and because URLs have length limits, this function constructs an array of
+     * query params that are certain to be short enough that each individual query will be a safe length, while still
+     * getting *all* of the requested data.
+     * @param keys The metadata keys we are requesting from the data sync service for the corresponding data source.
+     * @param prefixes The metadata prefixes we are requesting from the data sync service for the corresponding data source.
+     * @returns An array of query params such that each query param is short enough to be safe, but all params combined
+     *             cover all requested metadata.
+     */
+    getLengthLimitedQueryParams(keys, prefixes) {
+        const keyQueryParams = keys.map(encodeURIComponent).map((keyVal) => `metadataKeys=${keyVal}`);
+        const prefixQueryParams = prefixes.map(encodeURIComponent).map((prefixVal) => `metadataPrefixes=${prefixVal}`);
+        const fullQueryParams = [];
+        let currentQueryParam = '';
+        let delimiter = '?';
+        [...keyQueryParams, ...prefixQueryParams].forEach((param) => {
+            currentQueryParam = currentQueryParam.concat(`${delimiter}${param}`);
+            if (currentQueryParam.length > QueryParamLengthLimit) {
+                fullQueryParams.push(currentQueryParam);
+                currentQueryParam = '';
+                delimiter = '?';
+            }
+            else {
+                delimiter = '&';
+            }
+        });
+        if (currentQueryParam.length > 0) {
+            fullQueryParams.push(currentQueryParam);
         }
-        const encodedKeys = keys.map(encodeURIComponent);
-        const urlWithParams = `${this.metadataUrl}?metadataKeys=${encodedKeys.join('&metadataKeys=')}`;
-        const response = await this.fetchMethod(urlWithParams, {
+        return fullQueryParams;
+    }
+    async getResponseOrError(url) {
+        const response = await this.fetchMethod(url, {
             'method': 'GET',
             'headers': {
                 'Content-Type': 'application/json',
@@ -76,19 +106,57 @@ class DataSourceClient {
                 'Lucid-Api-Version': '1',
             },
         });
-        await (0, throwunsuccessful_1.throwUnsuccessful)(response, 'getting data source`s metadata');
+        try {
+            await (0, throwunsuccessful_1.throwUnsuccessful)(response, 'getting data source`s metadata');
+        }
+        catch (e) {
+            if (e instanceof dataconnector_1.DataConnectorResponseError) {
+                return { error: { url, body: e.response } };
+            }
+            throw e;
+        }
         const responseBody = await response.json();
         if ((0, validators_1.arrayValidator)(datasourcemetadatatypes_1.isSerializedMetadataRecord)(responseBody)) {
-            return responseBody.map((rawResponse) => {
-                return {
-                    key: rawResponse['key'],
-                    value: rawResponse['value'],
-                };
-            });
+            return {
+                value: responseBody.map((rawResponse) => {
+                    return {
+                        key: rawResponse['key'],
+                        value: rawResponse['value'],
+                    };
+                }),
+            };
         }
         else {
-            throw new dataconnector_1.DataConnectorResponseError(`Invalid response to query at '${urlWithParams}'! Expected an array of serialized metadata records`, JSON.stringify(responseBody));
+            return {
+                error: { url, body: responseBody },
+            };
+        }
+    }
+    /** @ignore until metadata endpoints are made part of the public API */
+    async getMetadata(keys, prefixes = []) {
+        if (this.metadataUrl == null) {
+            throw new Error('Attempted to access data sync service metadata endpoints in environment where the URL does not exist!');
+        }
+        const queryParamsList = this.getLengthLimitedQueryParams(keys, prefixes);
+        const urlsWithParams = queryParamsList.map((query) => `${this.metadataUrl}${query}`);
+        const responsePromises = urlsWithParams.map((url) => this.getResponseOrError(url));
+        const responses = await Promise.all(responsePromises);
+        const records = [];
+        const errors = [];
+        responses.forEach((response) => {
+            if ((0, result_1.isSuccess)(response)) {
+                records.push(...response.value);
+            }
+            else {
+                errors.push(response.error);
+            }
+        });
+        if (errors.length > 0) {
+            const message = `Error(s) received accessing URL(s): ${errors.map(({ url }) => url).join(',')}`;
+            const allBodies = errors.map(({ body }) => body);
+            throw new dataconnector_1.DataConnectorResponseError(message, JSON.stringify(allBodies));
         }
+        return records;
     }
     /** @ignore until metadata endpoints are made part of the public API */
     async patchMetadata(patches) {
@@ -156,8 +224,8 @@ class MockDataSourceClient extends DataSourceClient {
         return Promise.resolve(this.gotUpdate(request));
     }
     /** @ignore */
-    getMetadata(keys) {
-        return Promise.resolve(this.gotMetadata(keys));
+    getMetadata(keys, prefixes = []) {
+        return Promise.resolve(this.gotMetadata(keys, prefixes));
     }
     /** @ignore */
     patchMetadata(patches) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "lucid-extension-sdk",
-    "version": "0.0.149",
+    "version": "0.0.152",
     "description": "Utility classes for writing Lucid Software editor extensions",
     "main": "index.js",
     "types": "index.d.ts",