macrocosmos 2.1.0 → 2.1.4

package/dist/generated/gravity/v1/gravity.js

@@ -2,17 +2,115 @@
 // Code generated by protoc-gen-ts_proto. DO NOT EDIT.
 // versions:
 //   protoc-gen-ts_proto  v2.7.0
-//   protoc               v6.33.0
+//   protoc               v6.33.4
 // source: gravity/v1/gravity.proto
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AddPersistentGravityTaskRequest = exports.BuildAllDatasetsResponse = exports.BuildAllDatasetsRequest = exports.BuildDatasetResponse = exports.BuildDatasetRequest = exports.CreateGravityTaskResponse = exports.CreateGravityTaskRequest = exports.GetCrawlerResponse = exports.CompleteCrawlerRequest = exports.GetMarketplaceCrawlersResponse = exports.GetCrawlerRequest = exports.NotificationRequest = exports.GravityTask = exports.GetGravityTasksResponse = exports.GetGravityTasksRequest = exports.GravityMarketplaceTaskState = exports.GravityTaskState = exports.CrawlerState = exports.HfRepo = exports.CrawlerNotification = exports.PersistentTopicResponse = exports.PersistentTopic = exports.CrawlerCriteria = exports.InsertCrawlerCriteriaRequest = exports.GravityTaskRequest = exports.UpsertGravityTaskResponse = exports.UpsertGravityTaskRequest = exports.UpsertResponse = exports.UpsertCrawlerRequest = exports.GetPersistentGravityTasksResponse = exports.PersistentGravityTask = exports.Crawler = exports.GetPersistentDatasetWorkflowsResponse = exports.GetMarketplaceDatasetsRequest = exports.UpsertMarketplaceTaskMetadataRequest = exports.AddPersistentDatasetWorkflowsRequest = exports.PersistentDatasetWorkflow = exports.PublishDatasetRequest = exports.GetPopularTagsResponse = exports.PopularTag = exports.GetMarketplaceTaskSuggestionsResponse = exports.GetMarketplaceTaskSuggestionsRequest = exports.UpsertMarketplaceTaskSuggestionsRequest = exports.UpsertHotkeysRequest = exports.GetUserMarketplaceDatasetsResponse = exports.UserMarketplaceDataset = exports.BuyMarketplaceDatasetResponse = exports.BuyMarketplaceDatasetRequest = exports.GetHotkeysResponse = exports.protobufPackage = void 0;
-exports.GravityServiceClient = exports.GravityServiceService = exports.CrawlerDataForDD = exports.GetCrawlerDataForDDSubmissionResponse = exports.GetCrawlerDataForDDSubmissionRequest = exports.GetCrawlerHistoryResponse = exports.CrawlerCriteriaAndHistory = exports.CrawlerHistoryEntry = exports.GetCrawlerHistoryRequest = exports.GetGravityTaskDatasetFilesResponse = exports.DatasetFileWithId = exports.CrawlerRawMinerFilesResponse = exports.CrawlerDatasetFiles = exports.GetGravityTaskDatasetFilesRequest = exports.GetMarketplaceDatasetsResponse = exports.DatasetBillingCorrectionResponse = exports.DatasetBillingCorrectionRequest = exports.CancelDatasetResponse = exports.CancelDatasetRequest = exports.CancelGravityTaskResponse = exports.CancelGravityTaskRequest = exports.GetDatasetResponse = exports.GetDatasetRequest = exports.DatasetStep = exports.DatasetFile = exports.InsertDatasetFileRequest = exports.UpsertNebulaRequest = exports.UpsertDatasetRequest = exports.Dataset = exports.Nebula = void 0;
+exports.DatasetFile = exports.InsertDatasetFileRequest = exports.UpsertNebulaRequest = exports.UpsertDatasetRequest = exports.Dataset = exports.Nebula = exports.ChargeForDatasetRowsRequest = exports.BuildAllDatasetsResponse = exports.BuildAllDatasetsRequest = exports.BuildDatasetResponse = exports.BuildDatasetRequest = exports.CreateGravityTaskResponse = exports.CreateGravityTaskRequest = exports.GetCrawlerResponse = exports.CompleteCrawlerRequest = exports.GetMarketplaceCrawlersResponse = exports.GetCrawlerRequest = exports.NotificationRequest = exports.GravityTask = exports.GetGravityTasksResponse = exports.GetGravityTasksRequest = exports.GravityMarketplaceTaskState = exports.GravityTaskState = exports.CrawlerState = exports.HfRepo = exports.CrawlerNotification = exports.CrawlerCriteria = exports.InsertCrawlerCriteriaRequest = exports.GravityTaskRequest = exports.UpsertGravityTaskResponse = exports.UpsertGravityTaskRequest = exports.UpsertResponse = exports.UpsertCrawlerRequest = exports.Crawler = exports.GetMarketplaceDatasetsRequest = exports.UpsertMarketplaceTaskMetadataRequest = exports.PublishDatasetRequest = exports.GetPopularTagsResponse = exports.PopularTag = exports.GetMarketplaceTaskSuggestionsResponse = exports.GetMarketplaceTaskSuggestionsRequest = exports.UpsertMarketplaceTaskSuggestionsRequest = exports.UpsertHotkeysRequest = exports.GetUserMarketplaceDatasetsResponse = exports.UserMarketplaceDataset = exports.BuyMarketplaceDatasetResponse = exports.BuyMarketplaceDatasetRequest = exports.GetHotkeysResponse = exports.UpsertRawMinerFilesRequest = exports.protobufPackage = void 0;
+exports.GravityServiceClient = exports.GravityServiceService = exports.GetPreBuiltUserDatasetsResponse = exports.PreBuiltUserDataset = exports.GetPreBuiltUserDatasetsRequest = exports.UpsertPreBuiltUserDatasetsRequest = exports.ActiveUserTask = exports.ActiveUserCrawler = exports.GetActiveUserTasksResponse = exports.MarketplaceCrawlerDataForDDSubmission = exports.GetMarketplaceCrawlerDataForDDSubmissionResponse = exports.GetMarketplaceCrawlerDataForDDSubmissionRequest = exports.GetCrawlerHistoryResponse = exports.CrawlerCriteriaAndHistory = exports.CrawlerHistoryEntry = exports.GetCrawlerHistoryRequest = exports.GetGravityTaskDatasetFilesResponse = exports.DatasetFileWithId = exports.CrawlerRawMinerFilesResponse = exports.CrawlerDatasetFiles = exports.GetGravityTaskDatasetFilesRequest = exports.GetMarketplaceDatasetsResponse = exports.DatasetBillingCorrectionResponse = exports.DatasetBillingCorrectionRequest = exports.CancelDatasetResponse = exports.CancelDatasetRequest = exports.CancelGravityTaskResponse = exports.CancelGravityTaskRequest = exports.GetDatasetResponse = exports.GetDatasetRequest = exports.DatasetStep = void 0;
 /* eslint-disable */
 const wire_1 = require("@bufbuild/protobuf/wire");
 const grpc_js_1 = require("@grpc/grpc-js");
 const empty_1 = require("../../google/protobuf/empty");
 const timestamp_1 = require("../../google/protobuf/timestamp");
 exports.protobufPackage = "gravity.v1";
+function createBaseUpsertRawMinerFilesRequest() {
+    return { crawlerId: "", parquetPaths: [], pathSizes: [] };
+}
+exports.UpsertRawMinerFilesRequest = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.crawlerId !== "") {
+            writer.uint32(10).string(message.crawlerId);
+        }
+        for (const v of message.parquetPaths) {
+            writer.uint32(18).string(v);
+        }
+        writer.uint32(26).fork();
+        for (const v of message.pathSizes) {
+            writer.int64(v);
+        }
+        writer.join();
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseUpsertRawMinerFilesRequest();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.crawlerId = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.parquetPaths.push(reader.string());
+                    continue;
+                }
+                case 3: {
+                    if (tag === 24) {
+                        message.pathSizes.push(longToNumber(reader.int64()));
+                        continue;
+                    }
+                    if (tag === 26) {
+                        const end2 = reader.uint32() + reader.pos;
+                        while (reader.pos < end2) {
+                            message.pathSizes.push(longToNumber(reader.int64()));
+                        }
+                        continue;
+                    }
+                    break;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            crawlerId: isSet(object.crawlerId)
+                ? globalThis.String(object.crawlerId)
+                : "",
+            parquetPaths: globalThis.Array.isArray(object?.parquetPaths)
+                ? object.parquetPaths.map((e) => globalThis.String(e))
+                : [],
+            pathSizes: globalThis.Array.isArray(object?.pathSizes)
+                ? object.pathSizes.map((e) => globalThis.Number(e))
+                : [],
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.crawlerId !== "") {
+            obj.crawlerId = message.crawlerId;
+        }
+        if (message.parquetPaths?.length) {
+            obj.parquetPaths = message.parquetPaths;
+        }
+        if (message.pathSizes?.length) {
+            obj.pathSizes = message.pathSizes.map(e => Math.round(e));
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.UpsertRawMinerFilesRequest.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseUpsertRawMinerFilesRequest();
+        message.crawlerId = object.crawlerId ?? "";
+        message.parquetPaths = object.parquetPaths?.map(e => e) || [];
+        message.pathSizes = object.pathSizes?.map(e => e) || [];
+        return message;
+    },
+};
 function createBaseGetHotkeysResponse() {
     return { hotkeys: [] };
 }
@@ -796,132 +894,6 @@ exports.PublishDatasetRequest = {
         return message;
     },
 };
-function createBasePersistentDatasetWorkflow() {
-    return { datasetId: "", status: "" };
-}
-exports.PersistentDatasetWorkflow = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        if (message.datasetId !== "") {
-            writer.uint32(10).string(message.datasetId);
-        }
-        if (message.status !== "") {
-            writer.uint32(18).string(message.status);
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBasePersistentDatasetWorkflow();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.datasetId = reader.string();
-                    continue;
-                }
-                case 2: {
-                    if (tag !== 18) {
-                        break;
-                    }
-                    message.status = reader.string();
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            datasetId: isSet(object.datasetId)
-                ? globalThis.String(object.datasetId)
-                : "",
-            status: isSet(object.status) ? globalThis.String(object.status) : "",
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.datasetId !== "") {
-            obj.datasetId = message.datasetId;
-        }
-        if (message.status !== "") {
-            obj.status = message.status;
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.PersistentDatasetWorkflow.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBasePersistentDatasetWorkflow();
-        message.datasetId = object.datasetId ?? "";
-        message.status = object.status ?? "";
-        return message;
-    },
-};
-function createBaseAddPersistentDatasetWorkflowsRequest() {
-    return { datasetWorkflows: [] };
-}
-exports.AddPersistentDatasetWorkflowsRequest = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        for (const v of message.datasetWorkflows) {
-            exports.PersistentDatasetWorkflow.encode(v, writer.uint32(10).fork()).join();
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseAddPersistentDatasetWorkflowsRequest();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.datasetWorkflows.push(exports.PersistentDatasetWorkflow.decode(reader, reader.uint32()));
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            datasetWorkflows: globalThis.Array.isArray(object?.datasetWorkflows)
-                ? object.datasetWorkflows.map((e) => exports.PersistentDatasetWorkflow.fromJSON(e))
-                : [],
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.datasetWorkflows?.length) {
-            obj.datasetWorkflows = message.datasetWorkflows.map(e => exports.PersistentDatasetWorkflow.toJSON(e));
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.AddPersistentDatasetWorkflowsRequest.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBaseAddPersistentDatasetWorkflowsRequest();
-        message.datasetWorkflows =
-            object.datasetWorkflows?.map(e => exports.PersistentDatasetWorkflow.fromPartial(e)) || [];
-        return message;
-    },
-};
 function createBaseUpsertMarketplaceTaskMetadataRequest() {
     return {
         gravityTaskId: "",
@@ -1104,62 +1076,6 @@ exports.GetMarketplaceDatasetsRequest = {
         return message;
     },
 };
-function createBaseGetPersistentDatasetWorkflowsResponse() {
-    return { datasetWorkflows: [] };
-}
-exports.GetPersistentDatasetWorkflowsResponse = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        for (const v of message.datasetWorkflows) {
-            exports.PersistentDatasetWorkflow.encode(v, writer.uint32(10).fork()).join();
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseGetPersistentDatasetWorkflowsResponse();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.datasetWorkflows.push(exports.PersistentDatasetWorkflow.decode(reader, reader.uint32()));
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            datasetWorkflows: globalThis.Array.isArray(object?.datasetWorkflows)
-                ? object.datasetWorkflows.map((e) => exports.PersistentDatasetWorkflow.fromJSON(e))
-                : [],
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.datasetWorkflows?.length) {
-            obj.datasetWorkflows = message.datasetWorkflows.map(e => exports.PersistentDatasetWorkflow.toJSON(e));
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.GetPersistentDatasetWorkflowsResponse.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBaseGetPersistentDatasetWorkflowsResponse();
-        message.datasetWorkflows =
-            object.datasetWorkflows?.map(e => exports.PersistentDatasetWorkflow.fromPartial(e)) || [];
-        return message;
-    },
-};
 function createBaseCrawler() {
     return {
         crawlerId: "",
@@ -1349,23 +1265,23 @@ exports.Crawler = {
         return message;
     },
 };
-function createBasePersistentGravityTask() {
-    return { gravityTaskId: "", ingestDt: "" };
+function createBaseUpsertCrawlerRequest() {
+    return { gravityTaskId: "", crawler: undefined };
 }
-exports.PersistentGravityTask = {
+exports.UpsertCrawlerRequest = {
     encode(message, writer = new wire_1.BinaryWriter()) {
         if (message.gravityTaskId !== "") {
             writer.uint32(10).string(message.gravityTaskId);
         }
-        if (message.ingestDt !== "") {
-            writer.uint32(18).string(message.ingestDt);
+        if (message.crawler !== undefined) {
+            exports.Crawler.encode(message.crawler, writer.uint32(18).fork()).join();
         }
         return writer;
     },
     decode(input, length) {
         const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
         let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBasePersistentGravityTask();
+        const message = createBaseUpsertCrawlerRequest();
         while (reader.pos < end) {
             const tag = reader.uint32();
             switch (tag >>> 3) {
@@ -1380,7 +1296,7 @@ exports.PersistentGravityTask = {
                     if (tag !== 18) {
                         break;
                     }
-                    message.ingestDt = reader.string();
+                    message.crawler = exports.Crawler.decode(reader, reader.uint32());
                     continue;
                 }
             }
@@ -1396,9 +1312,9 @@ exports.PersistentGravityTask = {
             gravityTaskId: isSet(object.gravityTaskId)
                 ? globalThis.String(object.gravityTaskId)
                 : "",
-            ingestDt: isSet(object.ingestDt)
-                ? globalThis.String(object.ingestDt)
-                : "",
+            crawler: isSet(object.crawler)
+                ? exports.Crawler.fromJSON(object.crawler)
+                : undefined,
         };
     },
     toJSON(message) {
@@ -1406,35 +1322,38 @@ exports.PersistentGravityTask = {
         if (message.gravityTaskId !== "") {
             obj.gravityTaskId = message.gravityTaskId;
         }
-        if (message.ingestDt !== "") {
-            obj.ingestDt = message.ingestDt;
+        if (message.crawler !== undefined) {
+            obj.crawler = exports.Crawler.toJSON(message.crawler);
         }
         return obj;
     },
     create(base) {
-        return exports.PersistentGravityTask.fromPartial(base ?? {});
+        return exports.UpsertCrawlerRequest.fromPartial(base ?? {});
     },
     fromPartial(object) {
-        const message = createBasePersistentGravityTask();
+        const message = createBaseUpsertCrawlerRequest();
         message.gravityTaskId = object.gravityTaskId ?? "";
-        message.ingestDt = object.ingestDt ?? "";
+        message.crawler =
+            object.crawler !== undefined && object.crawler !== null
+                ? exports.Crawler.fromPartial(object.crawler)
+                : undefined;
         return message;
     },
 };
-function createBaseGetPersistentGravityTasksResponse() {
-    return { persistentGravityTasks: [] };
+function createBaseUpsertResponse() {
+    return { message: "" };
 }
-exports.GetPersistentGravityTasksResponse = {
+exports.UpsertResponse = {
     encode(message, writer = new wire_1.BinaryWriter()) {
-        for (const v of message.persistentGravityTasks) {
-            exports.PersistentGravityTask.encode(v, writer.uint32(10).fork()).join();
+        if (message.message !== "") {
+            writer.uint32(10).string(message.message);
         }
         return writer;
     },
     decode(input, length) {
         const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
         let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseGetPersistentGravityTasksResponse();
+        const message = createBaseUpsertResponse();
         while (reader.pos < end) {
             const tag = reader.uint32();
             switch (tag >>> 3) {
@@ -1442,7 +1361,7 @@ exports.GetPersistentGravityTasksResponse = {
                     if (tag !== 10) {
                         break;
                     }
-                    message.persistentGravityTasks.push(exports.PersistentGravityTask.decode(reader, reader.uint32()));
+                    message.message = reader.string();
                     continue;
                 }
             }
@@ -1455,149 +1374,18 @@ exports.GetPersistentGravityTasksResponse = {
     },
     fromJSON(object) {
         return {
-            persistentGravityTasks: globalThis.Array.isArray(object?.persistentGravityTasks)
-                ? object.persistentGravityTasks.map((e) => exports.PersistentGravityTask.fromJSON(e))
-                : [],
+            message: isSet(object.message) ? globalThis.String(object.message) : "",
         };
     },
     toJSON(message) {
         const obj = {};
-        if (message.persistentGravityTasks?.length) {
-            obj.persistentGravityTasks = message.persistentGravityTasks.map(e => exports.PersistentGravityTask.toJSON(e));
+        if (message.message !== "") {
+            obj.message = message.message;
         }
         return obj;
     },
     create(base) {
-        return exports.GetPersistentGravityTasksResponse.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBaseGetPersistentGravityTasksResponse();
-        message.persistentGravityTasks =
-            object.persistentGravityTasks?.map(e => exports.PersistentGravityTask.fromPartial(e)) || [];
-        return message;
-    },
-};
-function createBaseUpsertCrawlerRequest() {
-    return { gravityTaskId: "", crawler: undefined };
-}
-exports.UpsertCrawlerRequest = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        if (message.gravityTaskId !== "") {
-            writer.uint32(10).string(message.gravityTaskId);
-        }
-        if (message.crawler !== undefined) {
-            exports.Crawler.encode(message.crawler, writer.uint32(18).fork()).join();
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseUpsertCrawlerRequest();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.gravityTaskId = reader.string();
-                    continue;
-                }
-                case 2: {
-                    if (tag !== 18) {
-                        break;
-                    }
-                    message.crawler = exports.Crawler.decode(reader, reader.uint32());
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            gravityTaskId: isSet(object.gravityTaskId)
-                ? globalThis.String(object.gravityTaskId)
-                : "",
-            crawler: isSet(object.crawler)
-                ? exports.Crawler.fromJSON(object.crawler)
-                : undefined,
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.gravityTaskId !== "") {
-            obj.gravityTaskId = message.gravityTaskId;
-        }
-        if (message.crawler !== undefined) {
-            obj.crawler = exports.Crawler.toJSON(message.crawler);
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.UpsertCrawlerRequest.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBaseUpsertCrawlerRequest();
-        message.gravityTaskId = object.gravityTaskId ?? "";
-        message.crawler =
-            object.crawler !== undefined && object.crawler !== null
-                ? exports.Crawler.fromPartial(object.crawler)
-                : undefined;
-        return message;
-    },
-};
-function createBaseUpsertResponse() {
-    return { message: "" };
-}
-exports.UpsertResponse = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        if (message.message !== "") {
-            writer.uint32(10).string(message.message);
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseUpsertResponse();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.message = reader.string();
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            message: isSet(object.message) ? globalThis.String(object.message) : "",
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.message !== "") {
-            obj.message = message.message;
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.UpsertResponse.fromPartial(base ?? {});
+        return exports.UpsertResponse.fromPartial(base ?? {});
     },
     fromPartial(object) {
         const message = createBaseUpsertResponse();
@@ -2112,132 +1900,6 @@ exports.CrawlerCriteria = {
         return message;
     },
 };
-function createBasePersistentTopic() {
-    return { platform: "", topic: undefined };
-}
-exports.PersistentTopic = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        if (message.platform !== "") {
-            writer.uint32(10).string(message.platform);
-        }
-        if (message.topic !== undefined) {
-            writer.uint32(18).string(message.topic);
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBasePersistentTopic();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.platform = reader.string();
-                    continue;
-                }
-                case 2: {
-                    if (tag !== 18) {
-                        break;
-                    }
-                    message.topic = reader.string();
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            platform: isSet(object.platform)
-                ? globalThis.String(object.platform)
-                : "",
-            topic: isSet(object.topic) ? globalThis.String(object.topic) : undefined,
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.platform !== "") {
-            obj.platform = message.platform;
-        }
-        if (message.topic !== undefined) {
-            obj.topic = message.topic;
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.PersistentTopic.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBasePersistentTopic();
-        message.platform = object.platform ?? "";
-        message.topic = object.topic ?? undefined;
-        return message;
-    },
-};
-function createBasePersistentTopicResponse() {
-    return { persistentTopics: [] };
-}
-exports.PersistentTopicResponse = {
-    encode(message, writer = new wire_1.BinaryWriter()) {
-        for (const v of message.persistentTopics) {
-            exports.PersistentTopic.encode(v, writer.uint32(10).fork()).join();
-        }
-        return writer;
-    },
-    decode(input, length) {
-        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
-        let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBasePersistentTopicResponse();
-        while (reader.pos < end) {
-            const tag = reader.uint32();
-            switch (tag >>> 3) {
-                case 1: {
-                    if (tag !== 10) {
-                        break;
-                    }
-                    message.persistentTopics.push(exports.PersistentTopic.decode(reader, reader.uint32()));
-                    continue;
-                }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
-            reader.skip(tag & 7);
-        }
-        return message;
-    },
-    fromJSON(object) {
-        return {
-            persistentTopics: globalThis.Array.isArray(object?.persistentTopics)
-                ? object.persistentTopics.map((e) => exports.PersistentTopic.fromJSON(e))
-                : [],
-        };
-    },
-    toJSON(message) {
-        const obj = {};
-        if (message.persistentTopics?.length) {
-            obj.persistentTopics = message.persistentTopics.map(e => exports.PersistentTopic.toJSON(e));
-        }
-        return obj;
-    },
-    create(base) {
-        return exports.PersistentTopicResponse.fromPartial(base ?? {});
-    },
-    fromPartial(object) {
-        const message = createBasePersistentTopicResponse();
-        message.persistentTopics =
-            object.persistentTopics?.map(e => exports.PersistentTopic.fromPartial(e)) || [];
-        return message;
-    },
-};
 function createBaseCrawlerNotification() {
     return { to: "", link: "" };
 }
@@ -3660,7 +3322,7 @@ function createBaseBuildDatasetRequest() {
         crawlerId: "",
         notificationRequests: [],
         maxRows: 0,
-        isMarketplace: undefined,
+        isPeriodic: undefined,
     };
 }
 exports.BuildDatasetRequest = {
@@ -3674,8 +3336,8 @@ exports.BuildDatasetRequest = {
         if (message.maxRows !== 0) {
             writer.uint32(24).int64(message.maxRows);
         }
-        if (message.isMarketplace !== undefined) {
-            writer.uint32(32).bool(message.isMarketplace);
+        if (message.isPeriodic !== undefined) {
+            writer.uint32(32).bool(message.isPeriodic);
         }
         return writer;
     },
@@ -3711,7 +3373,7 @@ exports.BuildDatasetRequest = {
                     if (tag !== 32) {
                         break;
                     }
-                    message.isMarketplace = reader.bool();
+                    message.isPeriodic = reader.bool();
                     continue;
                 }
             }
@@ -3731,8 +3393,8 @@ exports.BuildDatasetRequest = {
                 ? object.notificationRequests.map((e) => exports.NotificationRequest.fromJSON(e))
                 : [],
             maxRows: isSet(object.maxRows) ? globalThis.Number(object.maxRows) : 0,
-            isMarketplace: isSet(object.isMarketplace)
-                ? globalThis.Boolean(object.isMarketplace)
+            isPeriodic: isSet(object.isPeriodic)
+                ? globalThis.Boolean(object.isPeriodic)
                 : undefined,
         };
     },
@@ -3747,8 +3409,8 @@ exports.BuildDatasetRequest = {
         if (message.maxRows !== 0) {
             obj.maxRows = Math.round(message.maxRows);
         }
-        if (message.isMarketplace !== undefined) {
-            obj.isMarketplace = message.isMarketplace;
+        if (message.isPeriodic !== undefined) {
+            obj.isPeriodic = message.isPeriodic;
         }
         return obj;
     },
@@ -3761,7 +3423,7 @@ exports.BuildDatasetRequest = {
         message.notificationRequests =
             object.notificationRequests?.map(e => exports.NotificationRequest.fromPartial(e)) || [];
         message.maxRows = object.maxRows ?? 0;
-        message.isMarketplace = object.isMarketplace ?? undefined;
+        message.isPeriodic = object.isPeriodic ?? undefined;
         return message;
     },
 };
@@ -3841,11 +3503,7 @@ exports.BuildDatasetResponse = {
     },
 };
 function createBaseBuildAllDatasetsRequest() {
-    return {
-        gravityTaskId: "",
-        buildCrawlersConfig: [],
-        isMarketplace: undefined,
-    };
+    return { gravityTaskId: "", buildCrawlersConfig: [] };
 }
 exports.BuildAllDatasetsRequest = {
     encode(message, writer = new wire_1.BinaryWriter()) {
@@ -3855,9 +3513,6 @@ exports.BuildAllDatasetsRequest = {
         for (const v of message.buildCrawlersConfig) {
             exports.BuildDatasetRequest.encode(v, writer.uint32(18).fork()).join();
         }
-        if (message.isMarketplace !== undefined) {
-            writer.uint32(24).bool(message.isMarketplace);
-        }
         return writer;
     },
     decode(input, length) {
@@ -3881,13 +3536,6 @@ exports.BuildAllDatasetsRequest = {
                     message.buildCrawlersConfig.push(exports.BuildDatasetRequest.decode(reader, reader.uint32()));
                     continue;
                 }
-                case 3: {
-                    if (tag !== 24) {
-                        break;
-                    }
-                    message.isMarketplace = reader.bool();
-                    continue;
-                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -3904,9 +3552,6 @@ exports.BuildAllDatasetsRequest = {
             buildCrawlersConfig: globalThis.Array.isArray(object?.buildCrawlersConfig)
                 ? object.buildCrawlersConfig.map((e) => exports.BuildDatasetRequest.fromJSON(e))
                 : [],
-            isMarketplace: isSet(object.isMarketplace)
-                ? globalThis.Boolean(object.isMarketplace)
-                : undefined,
         };
     },
     toJSON(message) {
@@ -3917,9 +3562,6 @@ exports.BuildAllDatasetsRequest = {
         if (message.buildCrawlersConfig?.length) {
             obj.buildCrawlersConfig = message.buildCrawlersConfig.map(e => exports.BuildDatasetRequest.toJSON(e));
         }
-        if (message.isMarketplace !== undefined) {
-            obj.isMarketplace = message.isMarketplace;
-        }
         return obj;
     },
     create(base) {
@@ -3930,7 +3572,6 @@ exports.BuildAllDatasetsRequest = {
         message.gravityTaskId = object.gravityTaskId ?? "";
         message.buildCrawlersConfig =
             object.buildCrawlersConfig?.map(e => exports.BuildDatasetRequest.fromPartial(e)) || [];
-        message.isMarketplace = object.isMarketplace ?? undefined;
         return message;
     },
 };
@@ -4006,20 +3647,23 @@ exports.BuildAllDatasetsResponse = {
         return message;
     },
 };
-function createBaseAddPersistentGravityTaskRequest() {
-    return { gravityTaskId: "" };
+function createBaseChargeForDatasetRowsRequest() {
+    return { crawlerId: "", rowCount: 0 };
 }
-exports.AddPersistentGravityTaskRequest = {
+exports.ChargeForDatasetRowsRequest = {
     encode(message, writer = new wire_1.BinaryWriter()) {
-        if (message.gravityTaskId !== "") {
-            writer.uint32(10).string(message.gravityTaskId);
+        if (message.crawlerId !== "") {
+            writer.uint32(10).string(message.crawlerId);
+        }
+        if (message.rowCount !== 0) {
+            writer.uint32(16).int64(message.rowCount);
         }
         return writer;
     },
     decode(input, length) {
         const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
         let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseAddPersistentGravityTaskRequest();
+        const message = createBaseChargeForDatasetRowsRequest();
         while (reader.pos < end) {
             const tag = reader.uint32();
             switch (tag >>> 3) {
@@ -4027,37 +3671,51 @@ exports.AddPersistentGravityTaskRequest = {
                     if (tag !== 10) {
                         break;
                     }
-                    message.gravityTaskId = reader.string();
+                    message.crawlerId = reader.string();
                     continue;
                 }
-            }
-            if ((tag & 7) === 4 || tag === 0) {
-                break;
-            }
+                case 2: {
+                    if (tag !== 16) {
+                        break;
+                    }
+                    message.rowCount = longToNumber(reader.int64());
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
             reader.skip(tag & 7);
         }
         return message;
     },
     fromJSON(object) {
         return {
-            gravityTaskId: isSet(object.gravityTaskId)
-                ? globalThis.String(object.gravityTaskId)
+            crawlerId: isSet(object.crawlerId)
+                ? globalThis.String(object.crawlerId)
                 : "",
+            rowCount: isSet(object.rowCount)
+                ? globalThis.Number(object.rowCount)
+                : 0,
         };
     },
     toJSON(message) {
         const obj = {};
-        if (message.gravityTaskId !== "") {
-            obj.gravityTaskId = message.gravityTaskId;
+        if (message.crawlerId !== "") {
+            obj.crawlerId = message.crawlerId;
+        }
+        if (message.rowCount !== 0) {
+            obj.rowCount = Math.round(message.rowCount);
         }
         return obj;
     },
     create(base) {
-        return exports.AddPersistentGravityTaskRequest.fromPartial(base ?? {});
+        return exports.ChargeForDatasetRowsRequest.fromPartial(base ?? {});
     },
     fromPartial(object) {
-        const message = createBaseAddPersistentGravityTaskRequest();
-        message.gravityTaskId = object.gravityTaskId ?? "";
+        const message = createBaseChargeForDatasetRowsRequest();
+        message.crawlerId = object.crawlerId ?? "";
+        message.rowCount = object.rowCount ?? 0;
         return message;
     },
 };
@@ -5451,7 +5109,7 @@ exports.CrawlerDatasetFiles = {
     },
 };
 function createBaseCrawlerRawMinerFilesResponse() {
-    return { crawlerId: "", s3Paths: [] };
+    return { crawlerId: "", s3Paths: [], fileSizeBytes: [] };
 }
 exports.CrawlerRawMinerFilesResponse = {
     encode(message, writer = new wire_1.BinaryWriter()) {
@@ -5461,6 +5119,11 @@ exports.CrawlerRawMinerFilesResponse = {
         for (const v of message.s3Paths) {
             writer.uint32(18).string(v);
         }
+        writer.uint32(26).fork();
+        for (const v of message.fileSizeBytes) {
+            writer.int64(v);
+        }
+        writer.join();
         return writer;
     },
     decode(input, length) {
@@ -5484,6 +5147,20 @@ exports.CrawlerRawMinerFilesResponse = {
                     message.s3Paths.push(reader.string());
                     continue;
                 }
+                case 3: {
+                    if (tag === 24) {
+                        message.fileSizeBytes.push(longToNumber(reader.int64()));
+                        continue;
+                    }
+                    if (tag === 26) {
+                        const end2 = reader.uint32() + reader.pos;
+                        while (reader.pos < end2) {
+                            message.fileSizeBytes.push(longToNumber(reader.int64()));
+                        }
+                        continue;
+                    }
+                    break;
+                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -5500,6 +5177,9 @@ exports.CrawlerRawMinerFilesResponse = {
             s3Paths: globalThis.Array.isArray(object?.s3Paths)
                 ? object.s3Paths.map((e) => globalThis.String(e))
                 : [],
+            fileSizeBytes: globalThis.Array.isArray(object?.fileSizeBytes)
+                ? object.fileSizeBytes.map((e) => globalThis.Number(e))
+                : [],
         };
     },
     toJSON(message) {
@@ -5510,6 +5190,9 @@ exports.CrawlerRawMinerFilesResponse = {
         if (message.s3Paths?.length) {
             obj.s3Paths = message.s3Paths;
         }
+        if (message.fileSizeBytes?.length) {
+            obj.fileSizeBytes = message.fileSizeBytes.map(e => Math.round(e));
+        }
         return obj;
     },
     create(base) {
@@ -5519,6 +5202,7 @@ exports.CrawlerRawMinerFilesResponse = {
         const message = createBaseCrawlerRawMinerFilesResponse();
         message.crawlerId = object.crawlerId ?? "";
         message.s3Paths = object.s3Paths?.map(e => e) || [];
+        message.fileSizeBytes = object.fileSizeBytes?.map(e => e) || [];
         return message;
     },
 };
@@ -6157,20 +5841,20 @@ exports.GetCrawlerHistoryResponse = {
         return message;
     },
 };
-function createBaseGetCrawlerDataForDDSubmissionRequest() {
-    return { dsns: [] };
+function createBaseGetMarketplaceCrawlerDataForDDSubmissionRequest() {
+    return { marketplaceUserId: "" };
 }
-exports.GetCrawlerDataForDDSubmissionRequest = {
+exports.GetMarketplaceCrawlerDataForDDSubmissionRequest = {
     encode(message, writer = new wire_1.BinaryWriter()) {
-        for (const v of message.dsns) {
-            writer.uint32(10).string(v);
+        if (message.marketplaceUserId !== "") {
+            writer.uint32(10).string(message.marketplaceUserId);
         }
         return writer;
     },
     decode(input, length) {
         const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
         let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseGetCrawlerDataForDDSubmissionRequest();
+        const message = createBaseGetMarketplaceCrawlerDataForDDSubmissionRequest();
         while (reader.pos < end) {
             const tag = reader.uint32();
             switch (tag >>> 3) {
@@ -6178,7 +5862,7 @@ exports.GetCrawlerDataForDDSubmissionRequest = {
                     if (tag !== 10) {
                         break;
                     }
-                    message.dsns.push(reader.string());
+                    message.marketplaceUserId = reader.string();
                     continue;
                 }
             }
@@ -6191,41 +5875,41 @@ exports.GetCrawlerDataForDDSubmissionRequest = {
     },
     fromJSON(object) {
         return {
-            dsns: globalThis.Array.isArray(object?.dsns)
-                ? object.dsns.map((e) => globalThis.String(e))
-                : [],
+            marketplaceUserId: isSet(object.marketplaceUserId)
+                ? globalThis.String(object.marketplaceUserId)
+                : "",
         };
     },
     toJSON(message) {
         const obj = {};
-        if (message.dsns?.length) {
-            obj.dsns = message.dsns;
+        if (message.marketplaceUserId !== "") {
+            obj.marketplaceUserId = message.marketplaceUserId;
         }
         return obj;
     },
     create(base) {
-        return exports.GetCrawlerDataForDDSubmissionRequest.fromPartial(base ?? {});
+        return exports.GetMarketplaceCrawlerDataForDDSubmissionRequest.fromPartial(base ?? {});
     },
     fromPartial(object) {
-        const message = createBaseGetCrawlerDataForDDSubmissionRequest();
-        message.dsns = object.dsns?.map(e => e) || [];
+        const message = createBaseGetMarketplaceCrawlerDataForDDSubmissionRequest();
+        message.marketplaceUserId = object.marketplaceUserId ?? "";
         return message;
     },
 };
-function createBaseGetCrawlerDataForDDSubmissionResponse() {
+function createBaseGetMarketplaceCrawlerDataForDDSubmissionResponse() {
     return { crawlers: [] };
 }
-exports.GetCrawlerDataForDDSubmissionResponse = {
+exports.GetMarketplaceCrawlerDataForDDSubmissionResponse = {
     encode(message, writer = new wire_1.BinaryWriter()) {
         for (const v of message.crawlers) {
-            exports.CrawlerDataForDD.encode(v, writer.uint32(10).fork()).join();
+            exports.MarketplaceCrawlerDataForDDSubmission.encode(v, writer.uint32(10).fork()).join();
         }
         return writer;
     },
     decode(input, length) {
         const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
         let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseGetCrawlerDataForDDSubmissionResponse();
+        const message = createBaseGetMarketplaceCrawlerDataForDDSubmissionResponse();
         while (reader.pos < end) {
             const tag = reader.uint32();
             switch (tag >>> 3) {
@@ -6233,7 +5917,7 @@ exports.GetCrawlerDataForDDSubmissionResponse = {
                     if (tag !== 10) {
                         break;
                     }
-                    message.crawlers.push(exports.CrawlerDataForDD.decode(reader, reader.uint32()));
+                    message.crawlers.push(exports.MarketplaceCrawlerDataForDDSubmission.decode(reader, reader.uint32()));
                     continue;
                 }
             }
@@ -6247,28 +5931,28 @@ exports.GetCrawlerDataForDDSubmissionResponse = {
     fromJSON(object) {
         return {
             crawlers: globalThis.Array.isArray(object?.crawlers)
-                ? object.crawlers.map((e) => exports.CrawlerDataForDD.fromJSON(e))
+                ? object.crawlers.map((e) => exports.MarketplaceCrawlerDataForDDSubmission.fromJSON(e))
                 : [],
         };
     },
     toJSON(message) {
         const obj = {};
         if (message.crawlers?.length) {
-            obj.crawlers = message.crawlers.map(e => exports.CrawlerDataForDD.toJSON(e));
+            obj.crawlers = message.crawlers.map(e => exports.MarketplaceCrawlerDataForDDSubmission.toJSON(e));
         }
         return obj;
     },
     create(base) {
-        return exports.GetCrawlerDataForDDSubmissionResponse.fromPartial(base ?? {});
+        return exports.GetMarketplaceCrawlerDataForDDSubmissionResponse.fromPartial(base ?? {});
     },
     fromPartial(object) {
-        const message = createBaseGetCrawlerDataForDDSubmissionResponse();
+        const message = createBaseGetMarketplaceCrawlerDataForDDSubmissionResponse();
         message.crawlers =
-            object.crawlers?.map(e => exports.CrawlerDataForDD.fromPartial(e)) || [];
+            object.crawlers?.map(e => exports.MarketplaceCrawlerDataForDDSubmission.fromPartial(e)) || [];
         return message;
     },
 };
-function createBaseCrawlerDataForDD() {
+function createBaseMarketplaceCrawlerDataForDDSubmission() {
     return {
         crawlerId: "",
         platform: "",
@@ -6276,9 +5960,18 @@ function createBaseCrawlerDataForDD() {
         keyword: undefined,
         postStartDatetime: undefined,
         postEndDatetime: undefined,
+        startTime: undefined,
+        deregistrationTime: undefined,
+        archiveTime: undefined,
+        status: "",
+        bytesCollected: 0,
+        recordsCollected: 0,
+        notificationTo: "",
+        notificationLink: "",
+        userId: "",
     };
 }
-exports.CrawlerDataForDD = {
+exports.MarketplaceCrawlerDataForDDSubmission = {
     encode(message, writer = new wire_1.BinaryWriter()) {
         if (message.crawlerId !== "") {
             writer.uint32(10).string(message.crawlerId);
@@ -6298,12 +5991,39 @@ exports.CrawlerDataForDD = {
         if (message.postEndDatetime !== undefined) {
             writer.uint32(50).string(message.postEndDatetime);
         }
+        if (message.startTime !== undefined) {
+            timestamp_1.Timestamp.encode(toTimestamp(message.startTime), writer.uint32(58).fork()).join();
+        }
+        if (message.deregistrationTime !== undefined) {
+            timestamp_1.Timestamp.encode(toTimestamp(message.deregistrationTime), writer.uint32(66).fork()).join();
+        }
+        if (message.archiveTime !== undefined) {
+            timestamp_1.Timestamp.encode(toTimestamp(message.archiveTime), writer.uint32(74).fork()).join();
+        }
+        if (message.status !== "") {
+            writer.uint32(82).string(message.status);
+        }
+        if (message.bytesCollected !== 0) {
+            writer.uint32(88).uint64(message.bytesCollected);
+        }
+        if (message.recordsCollected !== 0) {
+            writer.uint32(96).uint64(message.recordsCollected);
+        }
+        if (message.notificationTo !== "") {
+            writer.uint32(106).string(message.notificationTo);
+        }
+        if (message.notificationLink !== "") {
+            writer.uint32(114).string(message.notificationLink);
+        }
+        if (message.userId !== "") {
+            writer.uint32(122).string(message.userId);
+        }
         return writer;
     },
     decode(input, length) {
         const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
         let end = length === undefined ? reader.len : reader.pos + length;
-        const message = createBaseCrawlerDataForDD();
+        const message = createBaseMarketplaceCrawlerDataForDDSubmission();
         while (reader.pos < end) {
             const tag = reader.uint32();
             switch (tag >>> 3) {
@@ -6349,6 +6069,69 @@ exports.CrawlerDataForDD = {
                     message.postEndDatetime = reader.string();
                     continue;
                 }
+                case 7: {
+                    if (tag !== 58) {
+                        break;
+                    }
+                    message.startTime = fromTimestamp(timestamp_1.Timestamp.decode(reader, reader.uint32()));
+                    continue;
+                }
+                case 8: {
+                    if (tag !== 66) {
+                        break;
+                    }
+                    message.deregistrationTime = fromTimestamp(timestamp_1.Timestamp.decode(reader, reader.uint32()));
+                    continue;
+                }
+                case 9: {
+                    if (tag !== 74) {
+                        break;
+                    }
+                    message.archiveTime = fromTimestamp(timestamp_1.Timestamp.decode(reader, reader.uint32()));
+                    continue;
+                }
+                case 10: {
+                    if (tag !== 82) {
+                        break;
+                    }
+                    message.status = reader.string();
+                    continue;
+                }
+                case 11: {
+                    if (tag !== 88) {
+                        break;
+                    }
+                    message.bytesCollected = longToNumber(reader.uint64());
+                    continue;
+                }
+                case 12: {
+                    if (tag !== 96) {
+                        break;
+                    }
+                    message.recordsCollected = longToNumber(reader.uint64());
+                    continue;
+                }
+                case 13: {
+                    if (tag !== 106) {
+                        break;
+                    }
+                    message.notificationTo = reader.string();
+                    continue;
+                }
+                case 14: {
+                    if (tag !== 114) {
+                        break;
+                    }
+                    message.notificationLink = reader.string();
+                    continue;
+                }
+                case 15: {
+                    if (tag !== 122) {
+                        break;
+                    }
+                    message.userId = reader.string();
+                    continue;
+                }
             }
             if ((tag & 7) === 4 || tag === 0) {
                 break;
@@ -6365,7 +6148,9 @@ exports.CrawlerDataForDD = {
             platform: isSet(object.platform)
                 ? globalThis.String(object.platform)
                 : "",
-            topic: isSet(object.topic) ? globalThis.String(object.topic) : undefined,
+            topic: isSet(object.topic)
+                ? globalThis.String(object.topic)
+                : undefined,
             keyword: isSet(object.keyword)
                 ? globalThis.String(object.keyword)
                 : undefined,
@@ -6375,6 +6160,29 @@ exports.CrawlerDataForDD = {
             postEndDatetime: isSet(object.postEndDatetime)
                 ? globalThis.String(object.postEndDatetime)
                 : undefined,
+            startTime: isSet(object.startTime)
+                ? fromJsonTimestamp(object.startTime)
+                : undefined,
+            deregistrationTime: isSet(object.deregistrationTime)
+                ? fromJsonTimestamp(object.deregistrationTime)
+                : undefined,
+            archiveTime: isSet(object.archiveTime)
+                ? fromJsonTimestamp(object.archiveTime)
+                : undefined,
+            status: isSet(object.status) ? globalThis.String(object.status) : "",
+            bytesCollected: isSet(object.bytesCollected)
+                ? globalThis.Number(object.bytesCollected)
+                : 0,
+            recordsCollected: isSet(object.recordsCollected)
+                ? globalThis.Number(object.recordsCollected)
+                : 0,
+            notificationTo: isSet(object.notificationTo)
+                ? globalThis.String(object.notificationTo)
+                : "",
+            notificationLink: isSet(object.notificationLink)
+                ? globalThis.String(object.notificationLink)
+                : "",
+            userId: isSet(object.userId) ? globalThis.String(object.userId) : "",
         };
     },
     toJSON(message) {
@@ -6397,112 +6205,594 @@ exports.CrawlerDataForDD = {
         if (message.postEndDatetime !== undefined) {
             obj.postEndDatetime = message.postEndDatetime;
         }
+        if (message.startTime !== undefined) {
+            obj.startTime = message.startTime.toISOString();
+        }
+        if (message.deregistrationTime !== undefined) {
+            obj.deregistrationTime = message.deregistrationTime.toISOString();
+        }
+        if (message.archiveTime !== undefined) {
+            obj.archiveTime = message.archiveTime.toISOString();
+        }
+        if (message.status !== "") {
+            obj.status = message.status;
+        }
+        if (message.bytesCollected !== 0) {
+            obj.bytesCollected = Math.round(message.bytesCollected);
+        }
+        if (message.recordsCollected !== 0) {
+            obj.recordsCollected = Math.round(message.recordsCollected);
+        }
+        if (message.notificationTo !== "") {
+            obj.notificationTo = message.notificationTo;
+        }
+        if (message.notificationLink !== "") {
+            obj.notificationLink = message.notificationLink;
+        }
+        if (message.userId !== "") {
+            obj.userId = message.userId;
+        }
         return obj;
     },
     create(base) {
-        return exports.CrawlerDataForDD.fromPartial(base ?? {});
+        return exports.MarketplaceCrawlerDataForDDSubmission.fromPartial(base ?? {});
     },
     fromPartial(object) {
-        const message = createBaseCrawlerDataForDD();
+        const message = createBaseMarketplaceCrawlerDataForDDSubmission();
         message.crawlerId = object.crawlerId ?? "";
         message.platform = object.platform ?? "";
         message.topic = object.topic ?? undefined;
         message.keyword = object.keyword ?? undefined;
         message.postStartDatetime = object.postStartDatetime ?? undefined;
         message.postEndDatetime = object.postEndDatetime ?? undefined;
+        message.startTime = object.startTime ?? undefined;
+        message.deregistrationTime = object.deregistrationTime ?? undefined;
+        message.archiveTime = object.archiveTime ?? undefined;
+        message.status = object.status ?? "";
+        message.bytesCollected = object.bytesCollected ?? 0;
+        message.recordsCollected = object.recordsCollected ?? 0;
+        message.notificationTo = object.notificationTo ?? "";
+        message.notificationLink = object.notificationLink ?? "";
+        message.userId = object.userId ?? "";
         return message;
     },
 };
-exports.GravityServiceService = {
-    /** Lists all data collection tasks for a user */
-    getPopularTags: {
-        path: "/gravity.v1.GravityService/GetPopularTags",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
-        requestDeserialize: (value) => empty_1.Empty.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetPopularTagsResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetPopularTagsResponse.decode(value),
+function createBaseGetActiveUserTasksResponse() {
+    return { activeUserTasks: [] };
+}
+exports.GetActiveUserTasksResponse = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        for (const v of message.activeUserTasks) {
+            exports.ActiveUserTask.encode(v, writer.uint32(10).fork()).join();
+        }
+        return writer;
     },
-    /** Lists all data collection tasks for a user */
-    getGravityTasks: {
-        path: "/gravity.v1.GravityService/GetGravityTasks",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.GetGravityTasksRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.GetGravityTasksRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetGravityTasksResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetGravityTasksResponse.decode(value),
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseGetActiveUserTasksResponse();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.activeUserTasks.push(exports.ActiveUserTask.decode(reader, reader.uint32()));
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
     },
-    /** Get all marketplace crawlers */
-    getMarketplaceCrawlers: {
-        path: "/gravity.v1.GravityService/GetMarketplaceCrawlers",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
-        requestDeserialize: (value) => empty_1.Empty.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetMarketplaceCrawlersResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetMarketplaceCrawlersResponse.decode(value),
+    fromJSON(object) {
+        return {
+            activeUserTasks: globalThis.Array.isArray(object?.activeUserTasks)
+                ? object.activeUserTasks.map((e) => exports.ActiveUserTask.fromJSON(e))
+                : [],
+        };
     },
-    /** Gets raw miner files for a specific crawler */
-    getCrawlerRawMinerFiles: {
-        path: "/gravity.v1.GravityService/GetCrawlerRawMinerFiles",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.GetCrawlerRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.GetCrawlerRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.CrawlerRawMinerFilesResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.CrawlerRawMinerFilesResponse.decode(value),
+    toJSON(message) {
+        const obj = {};
+        if (message.activeUserTasks?.length) {
+            obj.activeUserTasks = message.activeUserTasks.map(e => exports.ActiveUserTask.toJSON(e));
+        }
+        return obj;
     },
-    /** Get the parent workflow id (the id of the ui workflow) for this crawler */
-    getCrawlerParentTaskId: {
-        path: "/gravity.v1.GravityService/GetCrawlerParentTaskId",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.GetCrawlerRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.GetCrawlerRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.CreateGravityTaskResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.CreateGravityTaskResponse.decode(value),
+    create(base) {
+        return exports.GetActiveUserTasksResponse.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseGetActiveUserTasksResponse();
+        message.activeUserTasks =
+            object.activeUserTasks?.map(e => exports.ActiveUserTask.fromPartial(e)) || [];
+        return message;
+    },
+};
+function createBaseActiveUserCrawler() {
+    return { crawlerId: "", rowCount: 0 };
+}
+exports.ActiveUserCrawler = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.crawlerId !== "") {
+            writer.uint32(10).string(message.crawlerId);
+        }
+        if (message.rowCount !== 0) {
+            writer.uint32(16).uint64(message.rowCount);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseActiveUserCrawler();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.crawlerId = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 16) {
+                        break;
+                    }
+                    message.rowCount = longToNumber(reader.uint64());
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            crawlerId: isSet(object.crawlerId)
+                ? globalThis.String(object.crawlerId)
+                : "",
+            rowCount: isSet(object.rowCount) ? globalThis.Number(object.rowCount) : 0,
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.crawlerId !== "") {
+            obj.crawlerId = message.crawlerId;
+        }
+        if (message.rowCount !== 0) {
+            obj.rowCount = Math.round(message.rowCount);
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.ActiveUserCrawler.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseActiveUserCrawler();
+        message.crawlerId = object.crawlerId ?? "";
+        message.rowCount = object.rowCount ?? 0;
+        return message;
+    },
+};
+function createBaseActiveUserTask() {
+    return { gravityTaskId: "", crawlers: [] };
+}
+exports.ActiveUserTask = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.gravityTaskId !== "") {
+            writer.uint32(10).string(message.gravityTaskId);
+        }
+        for (const v of message.crawlers) {
+            exports.ActiveUserCrawler.encode(v, writer.uint32(18).fork()).join();
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseActiveUserTask();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.gravityTaskId = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.crawlers.push(exports.ActiveUserCrawler.decode(reader, reader.uint32()));
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
     },
-    /** Add a persistent gravity task to the Gravity state DB */
-    addPersistentGravityTask: {
-        path: "/gravity.v1.GravityService/AddPersistentGravityTask",
+    fromJSON(object) {
+        return {
+            gravityTaskId: isSet(object.gravityTaskId)
+                ? globalThis.String(object.gravityTaskId)
+                : "",
+            crawlers: globalThis.Array.isArray(object?.crawlers)
+                ? object.crawlers.map((e) => exports.ActiveUserCrawler.fromJSON(e))
+                : [],
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.gravityTaskId !== "") {
+            obj.gravityTaskId = message.gravityTaskId;
+        }
+        if (message.crawlers?.length) {
+            obj.crawlers = message.crawlers.map(e => exports.ActiveUserCrawler.toJSON(e));
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.ActiveUserTask.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseActiveUserTask();
+        message.gravityTaskId = object.gravityTaskId ?? "";
+        message.crawlers =
+            object.crawlers?.map(e => exports.ActiveUserCrawler.fromPartial(e)) || [];
+        return message;
+    },
+};
+function createBaseUpsertPreBuiltUserDatasetsRequest() {
+    return { gravityTaskId: "", crawlerId: "", rowCount: 0 };
+}
+exports.UpsertPreBuiltUserDatasetsRequest = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.gravityTaskId !== "") {
+            writer.uint32(10).string(message.gravityTaskId);
+        }
+        if (message.crawlerId !== "") {
+            writer.uint32(18).string(message.crawlerId);
+        }
+        if (message.rowCount !== 0) {
+            writer.uint32(24).int64(message.rowCount);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseUpsertPreBuiltUserDatasetsRequest();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.gravityTaskId = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.crawlerId = reader.string();
+                    continue;
+                }
+                case 3: {
+                    if (tag !== 24) {
+                        break;
+                    }
+                    message.rowCount = longToNumber(reader.int64());
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            gravityTaskId: isSet(object.gravityTaskId)
+                ? globalThis.String(object.gravityTaskId)
+                : "",
+            crawlerId: isSet(object.crawlerId)
+                ? globalThis.String(object.crawlerId)
+                : "",
+            rowCount: isSet(object.rowCount)
+                ? globalThis.Number(object.rowCount)
+                : 0,
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.gravityTaskId !== "") {
+            obj.gravityTaskId = message.gravityTaskId;
+        }
+        if (message.crawlerId !== "") {
+            obj.crawlerId = message.crawlerId;
+        }
+        if (message.rowCount !== 0) {
+            obj.rowCount = Math.round(message.rowCount);
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.UpsertPreBuiltUserDatasetsRequest.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseUpsertPreBuiltUserDatasetsRequest();
+        message.gravityTaskId = object.gravityTaskId ?? "";
+        message.crawlerId = object.crawlerId ?? "";
+        message.rowCount = object.rowCount ?? 0;
+        return message;
+    },
+};
+function createBaseGetPreBuiltUserDatasetsRequest() {
+    return { gravityTaskId: "" };
+}
+exports.GetPreBuiltUserDatasetsRequest = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.gravityTaskId !== "") {
+            writer.uint32(10).string(message.gravityTaskId);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseGetPreBuiltUserDatasetsRequest();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.gravityTaskId = reader.string();
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            gravityTaskId: isSet(object.gravityTaskId)
+                ? globalThis.String(object.gravityTaskId)
+                : "",
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.gravityTaskId !== "") {
+            obj.gravityTaskId = message.gravityTaskId;
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.GetPreBuiltUserDatasetsRequest.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseGetPreBuiltUserDatasetsRequest();
+        message.gravityTaskId = object.gravityTaskId ?? "";
+        return message;
+    },
+};
+function createBasePreBuiltUserDataset() {
+    return { gravityTaskId: "", crawlerId: "", rowCount: 0 };
+}
+exports.PreBuiltUserDataset = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        if (message.gravityTaskId !== "") {
+            writer.uint32(10).string(message.gravityTaskId);
+        }
+        if (message.crawlerId !== "") {
+            writer.uint32(18).string(message.crawlerId);
+        }
+        if (message.rowCount !== 0) {
+            writer.uint32(24).int64(message.rowCount);
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBasePreBuiltUserDataset();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.gravityTaskId = reader.string();
+                    continue;
+                }
+                case 2: {
+                    if (tag !== 18) {
+                        break;
+                    }
+                    message.crawlerId = reader.string();
+                    continue;
+                }
+                case 3: {
+                    if (tag !== 24) {
+                        break;
+                    }
+                    message.rowCount = longToNumber(reader.int64());
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            gravityTaskId: isSet(object.gravityTaskId)
+                ? globalThis.String(object.gravityTaskId)
+                : "",
+            crawlerId: isSet(object.crawlerId)
+                ? globalThis.String(object.crawlerId)
+                : "",
+            rowCount: isSet(object.rowCount) ? globalThis.Number(object.rowCount) : 0,
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.gravityTaskId !== "") {
+            obj.gravityTaskId = message.gravityTaskId;
+        }
+        if (message.crawlerId !== "") {
+            obj.crawlerId = message.crawlerId;
+        }
+        if (message.rowCount !== 0) {
+            obj.rowCount = Math.round(message.rowCount);
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.PreBuiltUserDataset.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBasePreBuiltUserDataset();
+        message.gravityTaskId = object.gravityTaskId ?? "";
+        message.crawlerId = object.crawlerId ?? "";
+        message.rowCount = object.rowCount ?? 0;
+        return message;
+    },
+};
+function createBaseGetPreBuiltUserDatasetsResponse() {
+    return { datasets: [] };
+}
+exports.GetPreBuiltUserDatasetsResponse = {
+    encode(message, writer = new wire_1.BinaryWriter()) {
+        for (const v of message.datasets) {
+            exports.PreBuiltUserDataset.encode(v, writer.uint32(10).fork()).join();
+        }
+        return writer;
+    },
+    decode(input, length) {
+        const reader = input instanceof wire_1.BinaryReader ? input : new wire_1.BinaryReader(input);
+        let end = length === undefined ? reader.len : reader.pos + length;
+        const message = createBaseGetPreBuiltUserDatasetsResponse();
+        while (reader.pos < end) {
+            const tag = reader.uint32();
+            switch (tag >>> 3) {
+                case 1: {
+                    if (tag !== 10) {
+                        break;
+                    }
+                    message.datasets.push(exports.PreBuiltUserDataset.decode(reader, reader.uint32()));
+                    continue;
+                }
+            }
+            if ((tag & 7) === 4 || tag === 0) {
+                break;
+            }
+            reader.skip(tag & 7);
+        }
+        return message;
+    },
+    fromJSON(object) {
+        return {
+            datasets: globalThis.Array.isArray(object?.datasets)
+                ? object.datasets.map((e) => exports.PreBuiltUserDataset.fromJSON(e))
+                : [],
+        };
+    },
+    toJSON(message) {
+        const obj = {};
+        if (message.datasets?.length) {
+            obj.datasets = message.datasets.map(e => exports.PreBuiltUserDataset.toJSON(e));
+        }
+        return obj;
+    },
+    create(base) {
+        return exports.GetPreBuiltUserDatasetsResponse.fromPartial(base ?? {});
+    },
+    fromPartial(object) {
+        const message = createBaseGetPreBuiltUserDatasetsResponse();
+        message.datasets =
+            object.datasets?.map(e => exports.PreBuiltUserDataset.fromPartial(e)) || [];
+        return message;
+    },
+};
+exports.GravityServiceService = {
+    /** Lists all data collection tasks for a user */
+    getPopularTags: {
+        path: "/gravity.v1.GravityService/GetPopularTags",
         requestStream: false,
         responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.AddPersistentGravityTaskRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.AddPersistentGravityTaskRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.UpsertResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.UpsertResponse.decode(value),
+        requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
+        requestDeserialize: (value) => empty_1.Empty.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetPopularTagsResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetPopularTagsResponse.decode(value),
     },
-    /** Add a persistent dataset workflow to the Gravity state DB */
-    addPersistentDatasetWorkflows: {
-        path: "/gravity.v1.GravityService/AddPersistentDatasetWorkflows",
+    /** Lists all data collection tasks for a user */
+    getGravityTasks: {
+        path: "/gravity.v1.GravityService/GetGravityTasks",
         requestStream: false,
         responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.AddPersistentDatasetWorkflowsRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.AddPersistentDatasetWorkflowsRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.UpsertResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.UpsertResponse.decode(value),
+        requestSerialize: (value) => Buffer.from(exports.GetGravityTasksRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.GetGravityTasksRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetGravityTasksResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetGravityTasksResponse.decode(value),
     },
-    /** Retrieve recent persistent dataset workflows (last 7 days, non-Completed) */
-    getPersistentDatasetWorkflows: {
-        path: "/gravity.v1.GravityService/GetPersistentDatasetWorkflows",
+    /** Get all marketplace crawlers */
+    getMarketplaceCrawlers: {
+        path: "/gravity.v1.GravityService/GetMarketplaceCrawlers",
         requestStream: false,
         responseStream: false,
         requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
         requestDeserialize: (value) => empty_1.Empty.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetPersistentDatasetWorkflowsResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetPersistentDatasetWorkflowsResponse.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetMarketplaceCrawlersResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetMarketplaceCrawlersResponse.decode(value),
     },
-    /** Retrieve all persistent gravity tasks from the Gravity state DB */
-    getPersistentGravityTasks: {
-        path: "/gravity.v1.GravityService/GetPersistentGravityTasks",
+    /** Gets raw miner files for a specific crawler */
+    getCrawlerRawMinerFiles: {
+        path: "/gravity.v1.GravityService/GetCrawlerRawMinerFiles",
         requestStream: false,
         responseStream: false,
-        requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
-        requestDeserialize: (value) => empty_1.Empty.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetPersistentGravityTasksResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetPersistentGravityTasksResponse.decode(value),
+        requestSerialize: (value) => Buffer.from(exports.GetCrawlerRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.GetCrawlerRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.CrawlerRawMinerFilesResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.CrawlerRawMinerFilesResponse.decode(value),
+    },
+    /** Get the parent workflow id (the id of the ui workflow) for this crawler */
+    getCrawlerParentTaskId: {
+        path: "/gravity.v1.GravityService/GetCrawlerParentTaskId",
+        requestStream: false,
+        responseStream: false,
+        requestSerialize: (value) => Buffer.from(exports.GetCrawlerRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.GetCrawlerRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.CreateGravityTaskResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.CreateGravityTaskResponse.decode(value),
     },
     /** Get a single crawler by its ID */
     getCrawler: {
@@ -6554,6 +6844,16 @@ exports.GravityServiceService = {
         responseSerialize: (value) => Buffer.from(exports.CreateGravityTaskResponse.encode(value).finish()),
         responseDeserialize: (value) => exports.CreateGravityTaskResponse.decode(value),
     },
+    /** Gets all dataset files for a given marketplace gravity task (no user_id check, validates against marketplace tasks table) */
+    getGravityMarketplaceTaskDatasetFiles: {
+        path: "/gravity.v1.GravityService/GetGravityMarketplaceTaskDatasetFiles",
+        requestStream: false,
+        responseStream: false,
+        requestSerialize: (value) => Buffer.from(exports.GetGravityTaskDatasetFilesRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.GetGravityTaskDatasetFilesRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetGravityTaskDatasetFilesResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetGravityTaskDatasetFilesResponse.decode(value),
+    },
     /** Build a dataset for a single crawler */
     buildDataset: {
         path: "/gravity.v1.GravityService/BuildDataset",
@@ -6574,16 +6874,6 @@ exports.GravityServiceService = {
         responseSerialize: (value) => Buffer.from(exports.GetDatasetResponse.encode(value).finish()),
         responseDeserialize: (value) => exports.GetDatasetResponse.decode(value),
     },
-    /** Add a persistent topic to the Gravity state DB */
-    addPersistentTopic: {
-        path: "/gravity.v1.GravityService/AddPersistentTopic",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.PersistentTopic.encode(value).finish()),
-        requestDeserialize: (value) => exports.PersistentTopic.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.UpsertResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.UpsertResponse.decode(value),
-    },
     /** Cancel a gravity task and any crawlers associated with it */
     cancelGravityTask: {
         path: "/gravity.v1.GravityService/CancelGravityTask",
@@ -6634,16 +6924,6 @@ exports.GravityServiceService = {
         responseSerialize: (value) => Buffer.from(exports.GetGravityTaskDatasetFilesResponse.encode(value).finish()),
         responseDeserialize: (value) => exports.GetGravityTaskDatasetFilesResponse.decode(value),
     },
-    /** Gets all dataset files for a given persistent gravity task (no user_id check, validates against persistent tasks table) */
-    getGravityMarketplaceTaskDatasetFiles: {
-        path: "/gravity.v1.GravityService/GetGravityMarketplaceTaskDatasetFiles",
-        requestStream: false,
-        responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.GetGravityTaskDatasetFilesRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.GetGravityTaskDatasetFilesRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetGravityTaskDatasetFilesResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetGravityTaskDatasetFilesResponse.decode(value),
-    },
     /** Publishes a dataset into the Marketplace */
     publishDataset: {
         path: "/gravity.v1.GravityService/PublishDataset",
@@ -6655,14 +6935,24 @@ exports.GravityServiceService = {
         responseDeserialize: (value) => exports.UpsertResponse.decode(value),
     },
     /** Get crawler data for DD submission */
-    getCrawlerDataForDdSubmission: {
-        path: "/gravity.v1.GravityService/GetCrawlerDataForDDSubmission",
+    getActiveUserTasks: {
+        path: "/gravity.v1.GravityService/GetActiveUserTasks",
         requestStream: false,
         responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.GetCrawlerDataForDDSubmissionRequest.encode(value).finish()),
-        requestDeserialize: (value) => exports.GetCrawlerDataForDDSubmissionRequest.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.GetCrawlerDataForDDSubmissionResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.GetCrawlerDataForDDSubmissionResponse.decode(value),
+        requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
+        requestDeserialize: (value) => empty_1.Empty.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetActiveUserTasksResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetActiveUserTasksResponse.decode(value),
+    },
+    /** Get crawler data for DD submission for the marketplace user */
+    getMarketplaceCrawlerDataForDdSubmission: {
+        path: "/gravity.v1.GravityService/GetMarketplaceCrawlerDataForDDSubmission",
+        requestStream: false,
+        responseStream: false,
+        requestSerialize: (value) => Buffer.from(exports.GetMarketplaceCrawlerDataForDDSubmissionRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.GetMarketplaceCrawlerDataForDDSubmissionRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetMarketplaceCrawlerDataForDDSubmissionResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetMarketplaceCrawlerDataForDDSubmissionResponse.decode(value),
     },
     /** Upserts a crawler into the Gravity state DB */
     upsertCrawler: {
@@ -6734,15 +7024,25 @@ exports.GravityServiceService = {
         responseSerialize: (value) => Buffer.from(exports.BuildAllDatasetsResponse.encode(value).finish()),
         responseDeserialize: (value) => exports.BuildAllDatasetsResponse.decode(value),
     },
-    /** Gets all persistent topics from the Gravity state DB */
-    getPersistentTopics: {
-        path: "/gravity.v1.GravityService/GetPersistentTopics",
+    /** Builds datasets for multiple crawlers within a single gravity task periodically */
+    buildUserDatasetsPeriodically: {
+        path: "/gravity.v1.GravityService/BuildUserDatasetsPeriodically",
         requestStream: false,
         responseStream: false,
-        requestSerialize: (value) => Buffer.from(empty_1.Empty.encode(value).finish()),
-        requestDeserialize: (value) => empty_1.Empty.decode(value),
-        responseSerialize: (value) => Buffer.from(exports.PersistentTopicResponse.encode(value).finish()),
-        responseDeserialize: (value) => exports.PersistentTopicResponse.decode(value),
+        requestSerialize: (value) => Buffer.from(exports.BuildAllDatasetsRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.BuildAllDatasetsRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.BuildAllDatasetsResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.BuildAllDatasetsResponse.decode(value),
+    },
+    /** Charges a user for dataset rows */
+    chargeForDatasetRows: {
+        path: "/gravity.v1.GravityService/ChargeForDatasetRows",
+        requestStream: false,
+        responseStream: false,
+        requestSerialize: (value) => Buffer.from(exports.ChargeForDatasetRowsRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.ChargeForDatasetRowsRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.UpsertResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.UpsertResponse.decode(value),
     },
     /** Gets crawler history for a gravity task */
     getCrawlerHistory: {
@@ -6769,8 +7069,8 @@ exports.GravityServiceService = {
         path: "/gravity.v1.GravityService/UpsertRawMinerFiles",
         requestStream: false,
         responseStream: false,
-        requestSerialize: (value) => Buffer.from(exports.Crawler.encode(value).finish()),
-        requestDeserialize: (value) => exports.Crawler.decode(value),
+        requestSerialize: (value) => Buffer.from(exports.UpsertRawMinerFilesRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.UpsertRawMinerFilesRequest.decode(value),
         responseSerialize: (value) => Buffer.from(exports.UpsertResponse.encode(value).finish()),
         responseDeserialize: (value) => exports.UpsertResponse.decode(value),
     },
@@ -6814,6 +7114,26 @@ exports.GravityServiceService = {
         responseSerialize: (value) => Buffer.from(exports.GetUserMarketplaceDatasetsResponse.encode(value).finish()),
         responseDeserialize: (value) => exports.GetUserMarketplaceDatasetsResponse.decode(value),
     },
+    /** Upserts pre-built user dataset records */
+    upsertPreBuiltUserDatasets: {
+        path: "/gravity.v1.GravityService/UpsertPreBuiltUserDatasets",
+        requestStream: false,
+        responseStream: false,
+        requestSerialize: (value) => Buffer.from(exports.UpsertPreBuiltUserDatasetsRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.UpsertPreBuiltUserDatasetsRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.UpsertResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.UpsertResponse.decode(value),
+    },
+    /** Gets pre-built user dataset records for a gravity task */
+    getPreBuiltUserDatasets: {
+        path: "/gravity.v1.GravityService/GetPreBuiltUserDatasets",
+        requestStream: false,
+        responseStream: false,
+        requestSerialize: (value) => Buffer.from(exports.GetPreBuiltUserDatasetsRequest.encode(value).finish()),
+        requestDeserialize: (value) => exports.GetPreBuiltUserDatasetsRequest.decode(value),
+        responseSerialize: (value) => Buffer.from(exports.GetPreBuiltUserDatasetsResponse.encode(value).finish()),
+        responseDeserialize: (value) => exports.GetPreBuiltUserDatasetsResponse.decode(value),
+    },
 };
 exports.GravityServiceClient = (0, grpc_js_1.makeGenericClientConstructor)(exports.GravityServiceService, "gravity.v1.GravityService");
 function toTimestamp(date) {