macrocosmos 2.1.4 → 2.2.0

package/dist/__tests__/sn13/client.test.js

@@ -57,7 +57,7 @@ describe("Sn13Client", () => {
         expect(response.data.length).toBeLessThanOrEqual(request.limit);
         // Log response for debugging
         console.log("On-Demand Data Response:", response);
-    }, 60000); // Increase timeout to 60 seconds
+    }, 160000); // Increase timeout to 60 seconds
     it("should fetch on-demand data with keyword mode", async () => {
         // Create Sn13Client
         const client = new macrocosmos_1.Sn13Client({
@@ -82,7 +82,7 @@ describe("Sn13Client", () => {
         expect(response.data.length).toBeLessThanOrEqual(request.limit);
         // Log response for debugging
         console.log("On-Demand Data Response:", response);
-    }, 60000); // Increase timeout to 60 seconds
+    }, 160000); // Increase timeout to 60 seconds
     it("should fetch on-demand data with URL mode", async () => {
         // Create Sn13Client
         const client = new macrocosmos_1.Sn13Client({
@@ -103,5 +103,5 @@ describe("Sn13Client", () => {
         expect(Array.isArray(response.data)).toBe(true);
         // Log response for debugging
         console.log("On-Demand Data Response:", response);
-    }, 200000); // Increase timeout to 60 seconds
+    }, 1200000); // Increase timeout to 60 seconds
 });

package/dist/generated/billing/v1/billing.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: billing/v1/billing.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BillingServiceClient = exports.BillingServiceService = exports.UserHasEnoughAllowanceAndCreditsResponse = exports.UserHasEnoughAllowanceAndCreditsRequest = exports.ChargeUserForUsageResponse = exports.ChargeUserForUsageRequest = exports.GetUsageResponse = exports.SubscriptionInfo = exports.BillingRate = exports.GetUsageRequest = exports.protobufPackage = void 0;

package/dist/generated/google/protobuf/empty.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: google/protobuf/empty.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Empty = exports.protobufPackage = void 0;

package/dist/generated/google/protobuf/struct.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: google/protobuf/struct.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ListValue = exports.Value = exports.Struct_FieldsEntry = exports.Struct = exports.NullValue = exports.protobufPackage = void 0;

package/dist/generated/google/protobuf/timestamp.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: google/protobuf/timestamp.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Timestamp = exports.protobufPackage = void 0;

package/dist/generated/gravity/v1/gravity.d.ts

@@ -122,6 +122,8 @@ export interface Crawler {
     datasetWorkflows: string[];
     /** parquet_paths: the paths to the raw miner files collected */
     parquetPaths: string[];
+    /** pre_built_row_count: the number of rows in the pre-built user dataset (if present) */
+    preBuiltRowCount?: number | undefined;
 }
 /** UpsertCrawlerRequest for upserting a crawler and its criteria */
 export interface UpsertCrawlerRequest {

package/dist/generated/gravity/v1/gravity.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: gravity/v1/gravity.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatasetFile = exports.InsertDatasetFileRequest = exports.UpsertNebulaRequest = exports.UpsertDatasetRequest = exports.Dataset = exports.Nebula = exports.ChargeForDatasetRowsRequest = exports.BuildAllDatasetsResponse = exports.BuildAllDatasetsRequest = exports.BuildDatasetResponse = exports.BuildDatasetRequest = exports.CreateGravityTaskResponse = exports.CreateGravityTaskRequest = exports.GetCrawlerResponse = exports.CompleteCrawlerRequest = exports.GetMarketplaceCrawlersResponse = exports.GetCrawlerRequest = exports.NotificationRequest = exports.GravityTask = exports.GetGravityTasksResponse = exports.GetGravityTasksRequest = exports.GravityMarketplaceTaskState = exports.GravityTaskState = exports.CrawlerState = exports.HfRepo = exports.CrawlerNotification = exports.CrawlerCriteria = exports.InsertCrawlerCriteriaRequest = exports.GravityTaskRequest = exports.UpsertGravityTaskResponse = exports.UpsertGravityTaskRequest = exports.UpsertResponse = exports.UpsertCrawlerRequest = exports.Crawler = exports.GetMarketplaceDatasetsRequest = exports.UpsertMarketplaceTaskMetadataRequest = exports.PublishDatasetRequest = exports.GetPopularTagsResponse = exports.PopularTag = exports.GetMarketplaceTaskSuggestionsResponse = exports.GetMarketplaceTaskSuggestionsRequest = exports.UpsertMarketplaceTaskSuggestionsRequest = exports.UpsertHotkeysRequest = exports.GetUserMarketplaceDatasetsResponse = exports.UserMarketplaceDataset = exports.BuyMarketplaceDatasetResponse = exports.BuyMarketplaceDatasetRequest = exports.GetHotkeysResponse = exports.UpsertRawMinerFilesRequest = exports.protobufPackage = void 0;
@@ -1086,6 +1086,7 @@ function createBaseCrawler() {
         state: undefined,
         datasetWorkflows: [],
         parquetPaths: [],
+        preBuiltRowCount: undefined,
     };
 }
 exports.Crawler = {
@@ -1114,6 +1115,9 @@ exports.Crawler = {
         for (const v of message.parquetPaths) {
             writer.uint32(66).string(v);
         }
+        if (message.preBuiltRowCount !== undefined) {
+            writer.uint32(72).int64(message.preBuiltRowCount);
+        }
         return writer;
     },
     decode(input, length) {
@@ -1179,6 +1183,13 @@ exports.Crawler = {
                     message.parquetPaths.push(reader.string());
                     continue;
                 }
+                case 9: {
+                    if (tag !== 72) {
+                        break;
+                    }
+                    message.preBuiltRowCount = longToNumber(reader.int64());
+                    continue;
+                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -1213,6 +1224,9 @@ exports.Crawler = {
             parquetPaths: globalThis.Array.isArray(object?.parquetPaths)
                 ? object.parquetPaths.map((e) => globalThis.String(e))
                 : [],
+            preBuiltRowCount: isSet(object.preBuiltRowCount)
+                ? globalThis.Number(object.preBuiltRowCount)
+                : undefined,
         };
     },
     toJSON(message) {
@@ -1241,6 +1255,9 @@ exports.Crawler = {
         if (message.parquetPaths?.length) {
             obj.parquetPaths = message.parquetPaths;
         }
+        if (message.preBuiltRowCount !== undefined) {
+            obj.preBuiltRowCount = Math.round(message.preBuiltRowCount);
+        }
         return obj;
     },
     create(base) {
@@ -1262,6 +1279,7 @@ exports.Crawler = {
                 : undefined;
         message.datasetWorkflows = object.datasetWorkflows?.map(e => e) || [];
         message.parquetPaths = object.parquetPaths?.map(e => e) || [];
+        message.preBuiltRowCount = object.preBuiltRowCount ?? undefined;
         return message;
     },
 };

package/dist/generated/logger/v1/logger.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: logger/v1/logger.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.LoggerServiceClient = exports.LoggerServiceService = exports.StoreRecordBatchRequest = exports.Record = exports.CreateRunRequest = exports.Ack = exports.protobufPackage = void 0;

package/dist/generated/sn13/v1/sn13_validator.js

@@ -2,7 +2,7 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.4
+//   protoc               v6.33.5
 // source: sn13/v1/sn13_validator.proto
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Sn13ServiceClient = exports.Sn13ServiceService = exports.OnDemandDataResponse = exports.OnDemandDataRequest = exports.ValidateRedditTopicResponse = exports.ValidateRedditTopicRequest = exports.ListTopicsResponse = exports.ListTopicsResponseDetail = exports.ListTopicsRequest = exports.protobufPackage = void 0;

package/dist/lib/gravity/Client.d.ts

@@ -1,9 +1,10 @@
-import { GravityServiceClient, GetGravityTasksRequest, GetGravityTasksResponse, GetCrawlerRequest, GetCrawlerResponse, CreateGravityTaskRequest as GeneratedCreateGravityTaskRequest, CreateGravityTaskResponse, BuildDatasetRequest as GeneratedBuildDatasetRequest, BuildDatasetResponse, GetDatasetRequest, GetDatasetResponse, CancelGravityTaskRequest, CancelGravityTaskResponse, CancelDatasetRequest, CancelDatasetResponse, Crawler, CrawlerCriteria, CrawlerNotification, HfRepo, CrawlerState, GravityTaskState, NotificationRequest, Dataset, DatasetFile, DatasetStep, Nebula, GetGravityTaskDatasetFilesResponse, GetGravityTaskDatasetFilesRequest, BuildAllDatasetsResponse, BuildAllDatasetsRequest } from "../../generated/gravity/v1/gravity";
+import { GravityServiceClient, GetGravityTasksRequest, GetGravityTasksResponse, GetCrawlerRequest, GetCrawlerResponse, CreateGravityTaskRequest as GeneratedCreateGravityTaskRequest, CreateGravityTaskResponse, BuildDatasetRequest as GeneratedBuildDatasetRequest, BuildDatasetResponse, GetDatasetRequest, GetDatasetResponse, CancelGravityTaskRequest, CancelGravityTaskResponse, CancelDatasetRequest, CancelDatasetResponse, Crawler, CrawlerCriteria, CrawlerNotification, HfRepo, CrawlerState, GravityTaskState, NotificationRequest, Dataset, DatasetFile, DatasetStep, Nebula, GetGravityTaskDatasetFilesResponse, GetGravityTaskDatasetFilesRequest, BuildAllDatasetsResponse, BuildAllDatasetsRequest, GetMarketplaceDatasetsRequest, GetMarketplaceDatasetsResponse, GetCrawlerHistoryRequest, GetCrawlerHistoryResponse, GetUserMarketplaceDatasetsResponse, GetMarketplaceTaskSuggestionsRequest, GetPopularTagsResponse, BuyMarketplaceDatasetRequest, BuyMarketplaceDatasetResponse, UserMarketplaceDataset, PopularTag, CrawlerHistoryEntry, CrawlerCriteriaAndHistory } from "../../generated/gravity/v1/gravity";
+import { Empty } from "../../generated/google/protobuf/empty";
 import { BaseClient, BaseClientOptions } from "../BaseClient";
 import { MarkFieldsOptional } from "../util.types";
 type CreateGravityTaskRequest = MarkFieldsOptional<GeneratedCreateGravityTaskRequest, "notificationRequests">;
 type BuildDatasetRequest = MarkFieldsOptional<GeneratedBuildDatasetRequest, "notificationRequests">;
-export type { GetGravityTasksRequest, GetGravityTasksResponse, GetCrawlerRequest, GetCrawlerResponse, CreateGravityTaskRequest, CreateGravityTaskResponse, BuildDatasetRequest, BuildDatasetResponse, GetDatasetRequest, GetDatasetResponse, CancelGravityTaskRequest, CancelGravityTaskResponse, CancelDatasetRequest, CancelDatasetResponse, Crawler, CrawlerCriteria, CrawlerNotification, HfRepo, CrawlerState, GravityTaskState, NotificationRequest, Dataset, DatasetFile, DatasetStep, Nebula, };
+export type { GetGravityTasksRequest, GetGravityTasksResponse, GetCrawlerRequest, GetCrawlerResponse, CreateGravityTaskRequest, CreateGravityTaskResponse, BuildDatasetRequest, BuildDatasetResponse, GetDatasetRequest, GetDatasetResponse, CancelGravityTaskRequest, CancelGravityTaskResponse, CancelDatasetRequest, CancelDatasetResponse, Crawler, CrawlerCriteria, CrawlerNotification, HfRepo, CrawlerState, GravityTaskState, NotificationRequest, Dataset, DatasetFile, DatasetStep, Nebula, GetMarketplaceDatasetsRequest, GetMarketplaceDatasetsResponse, GetCrawlerHistoryRequest, GetCrawlerHistoryResponse, GetUserMarketplaceDatasetsResponse, GetMarketplaceTaskSuggestionsRequest, GetPopularTagsResponse, BuyMarketplaceDatasetRequest, BuyMarketplaceDatasetResponse, UserMarketplaceDataset, PopularTag, CrawlerHistoryEntry, CrawlerCriteriaAndHistory, Empty, };
 /**
  * Client for interacting with the Gravity API
  * Provides gRPC interface for data collection and dataset management
@@ -41,5 +42,34 @@ export declare class GravityClient extends BaseClient {
      */
     cancelDataset: (params: CancelDatasetRequest) => Promise<CancelDatasetResponse>;
     getGravityTaskDatasetFiles: (params: GetGravityTaskDatasetFilesRequest) => Promise<GetGravityTaskDatasetFilesResponse>;
+    /**
+     * Gets the available datasets for use in Dataset Marketplace
+     */
+    getMarketplaceDatasets: (params: GetMarketplaceDatasetsRequest) => Promise<GetMarketplaceDatasetsResponse>;
     buildAllDatasets: (params: BuildAllDatasetsRequest) => Promise<BuildAllDatasetsResponse>;
+    /**
+     * Gets all dataset files for a given marketplace gravity task
+     * (no user_id check, validates against marketplace tasks table)
+     */
+    getGravityMarketplaceTaskDatasetFiles: (params: GetGravityTaskDatasetFilesRequest) => Promise<GetGravityTaskDatasetFilesResponse>;
+    /**
+     * Gets crawler history for a given gravity task
+     */
+    getCrawlerHistory: (params: GetCrawlerHistoryRequest) => Promise<GetCrawlerHistoryResponse>;
+    /**
+     * Get all marketplace datasets owned by the authenticated user
+     */
+    getUserMarketplaceDatasets: () => Promise<GetUserMarketplaceDatasetsResponse>;
+    /**
+     * Get marketplace task suggestions for a given gravity task
+     */
+    getMarketplaceTaskSuggestions: (params: GetMarketplaceTaskSuggestionsRequest) => Promise<GetMarketplaceDatasetsResponse>;
+    /**
+     * Gets popular tags across all gravity tasks
+     */
+    getPopularTags: () => Promise<GetPopularTagsResponse>;
+    /**
+     * Purchase a marketplace dataset
+     */
+    buyMarketplaceDataset: (params: BuyMarketplaceDatasetRequest) => Promise<BuyMarketplaceDatasetResponse>;
 }

package/dist/lib/gravity/Client.js

@@ -223,6 +223,33 @@ class GravityClient extends BaseClient_1.BaseClient {
                 });
             });
         };
+        /**
+         * Gets the available datasets for use in Dataset Marketplace
+         */
+        this.getMarketplaceDatasets = (params) => {
+            const client = this.createGrpcClient();
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.getMarketplaceDatasets(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.getMarketplaceDatasets(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
         this.buildAllDatasets = (params) => {
             const client = this.createGrpcClient();
             if (this.isSecure()) {
@@ -247,6 +274,171 @@ class GravityClient extends BaseClient_1.BaseClient {
                 });
             });
         };
+        /**
+         * Gets all dataset files for a given marketplace gravity task
+         * (no user_id check, validates against marketplace tasks table)
+         */
+        this.getGravityMarketplaceTaskDatasetFiles = (params) => {
+            const client = this.createGrpcClient();
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.getGravityMarketplaceTaskDatasetFiles(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.getGravityMarketplaceTaskDatasetFiles(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
+        /**
+         * Gets crawler history for a given gravity task
+         */
+        this.getCrawlerHistory = (params) => {
+            const client = this.createGrpcClient();
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.getCrawlerHistory(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.getCrawlerHistory(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
+        /**
+         * Get all marketplace datasets owned by the authenticated user
+         */
+        this.getUserMarketplaceDatasets = () => {
+            const client = this.createGrpcClient();
+            const params = {};
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.getUserMarketplaceDatasets(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.getUserMarketplaceDatasets(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
+        /**
+         * Get marketplace task suggestions for a given gravity task
+         */
+        this.getMarketplaceTaskSuggestions = (params) => {
+            const client = this.createGrpcClient();
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.getMarketplaceTaskSuggestions(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.getMarketplaceTaskSuggestions(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
+        /**
+         * Gets popular tags across all gravity tasks
+         */
+        this.getPopularTags = () => {
+            const client = this.createGrpcClient();
+            const params = {};
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.getPopularTags(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.getPopularTags(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
+        /**
+         * Purchase a marketplace dataset
+         */
+        this.buyMarketplaceDataset = (params) => {
+            const client = this.createGrpcClient();
+            if (this.isSecure()) {
+                return new Promise((resolve, reject) => {
+                    client.buyMarketplaceDataset(params, (error, response) => {
+                        if (error) {
+                            reject(error);
+                            return;
+                        }
+                        resolve(response);
+                    });
+                });
+            }
+            const metadata = this.createAuthMetadata();
+            return new Promise((resolve, reject) => {
+                client.buyMarketplaceDataset(params, metadata, (error, response) => {
+                    if (error) {
+                        reject(error);
+                        return;
+                    }
+                    resolve(response);
+                });
+            });
+        };
         this._grpcClient = grpcClient;
     }
     createGrpcClient() {

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "2.1.4";
+export declare const VERSION = "2.2.0";

package/dist/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VERSION = void 0;
-exports.VERSION = "2.1.4";
+exports.VERSION = "2.2.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "macrocosmos",
-  "version": "2.1.4",
+  "version": "2.2.0",
   "description": "TypeScript SDK package for Macrocosmos",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",