azure-kusto-ingest 5.2.3 → 6.0.1

package/types/src/ingestionResult.d.ts

@@ -0,0 +1,41 @@
+import { TableClient, TableEntity } from "@azure/data-tables";
+export interface IngestionResult {
+    getIngestionStatusCollection(): Promise<IngestionStatus>;
+}
+export declare const putRecordInTable: (tableClient: TableClient, entity: TableEntity<IngestionStatus>) => Promise<void>;
+export declare enum OperationStatus {
+    Pending = "Pending",
+    Succeede = "Succeeded",
+    Failed = "Failed",
+    Queued = "Queued",
+    Skipped = "Skipped",
+    PartiallySucceeded = "PartiallySucceeded"
+}
+export declare class TableReportIngestionResult implements IngestionResult {
+    private ingestionStatusInTableDescription;
+    tableClient: TableClient | null;
+    constructor(ingestionStatusInTableDescription: IngestionStatusInTableDescription, tableClient?: TableClient | null);
+    getIngestionStatusCollection(): Promise<IngestionStatus>;
+}
+export declare class IngestionStatusResult implements IngestionResult {
+    private ingestionStatus;
+    constructor(ingestionStatus: IngestionStatus);
+    getIngestionStatusCollection(): Promise<IngestionStatus>;
+}
+export declare class IngestionStatusInTableDescription {
+    tableConnectionString: string;
+    partitionKey: string;
+    rowKey: string;
+    constructor(tableConnectionString: string, partitionKey: string, rowKey: string);
+}
+export interface IngestionStatus {
+    Timestamp: string;
+    Status: OperationStatus;
+    IngestionSourceId: string;
+    IngestionSourcePath: string;
+    Database: string;
+    Table: string;
+    UpdatedOn: string;
+    Details: string;
+}
+//# sourceMappingURL=ingestionResult.d.ts.map

package/types/src/managedStreamingIngestClient.d.ts

@@ -1,11 +1,11 @@
 /// <reference types="node" />
 import { IngestionPropertiesInput } from "./ingestionProperties";
-import { QueueSendMessageResponse } from "@azure/storage-queue";
 import { KustoConnectionStringBuilder, KustoResponseDataSet } from "azure-kusto-data";
 import { Readable } from "stream";
 import { AbstractKustoClient } from "./abstractKustoClient";
 import { AbstractDescriptor, BlobDescriptor, StreamDescriptor } from "./descriptors";
 import { FileDescriptor } from "./fileDescriptor";
+import { IngestionResult } from "./ingestionResult";
 declare class KustoManagedStreamingIngestClient extends AbstractKustoClient {
     private streamingIngestClient;
     private queuedIngestClient;
@@ -29,16 +29,16 @@ declare class KustoManagedStreamingIngestClient extends AbstractKustoClient {
      * @param defaultProps The default ingestion properties.
      */
     static fromEngineConnectionString(engineConnectionString: KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput): KustoManagedStreamingIngestClient;
-    constructor(engineKcsb: string | KustoConnectionStringBuilder, dmKcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
+    constructor(engineKcsb: string | KustoConnectionStringBuilder, dmKcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */
-    ingestFromStream(stream: StreamDescriptor | Readable | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any>;
+    ingestFromStream(stream: StreamDescriptor | Readable | ArrayBuffer, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<KustoResponseDataSet | IngestionResult>;
     /**
      * Use string for Node.js and Blob in browser
      */
-    ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet | QueueSendMessageResponse>;
-    ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any>;
+    ingestFromFile(file: FileDescriptor | string | Blob, ingestionProperties?: IngestionPropertiesInput): Promise<KustoResponseDataSet | IngestionResult>;
+    ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<KustoResponseDataSet | IngestionResult>;
     streamWithRetries(length: number, descriptor: AbstractDescriptor, props?: IngestionPropertiesInput, clientRequestId?: string, stream?: Readable | ArrayBuffer): Promise<any>;
     close(): void;
 }

package/types/src/rankedStorageAccount.d.ts

@@ -0,0 +1,15 @@
+export declare class RankedStorageAccount {
+    private accountName;
+    private numberOfBuckets;
+    private bucketDuration;
+    private timeProvider;
+    private buckets;
+    private lastUpdateTime;
+    private currentBucketIndex;
+    constructor(accountName: string, numberOfBuckets: number, bucketDuration: number, timeProvider: () => number);
+    logResult(success: boolean): void;
+    getAccountName(): string;
+    adjustForTimePassed(): number;
+    getRank(): number;
+}
+//# sourceMappingURL=rankedStorageAccount.d.ts.map

package/types/src/rankedStorageAccountSet.d.ts

@@ -0,0 +1,18 @@
+import { RankedStorageAccount } from "./rankedStorageAccount";
+export declare class RankedStorageAccountSet {
+    private numberOfBuckets;
+    private bucketDuration;
+    private tiers;
+    private timeProvider;
+    static readonly DefaultNumberOfBuckets: number;
+    static readonly DefaultBucketDurationInSeconds: number;
+    static readonly DefaultTiers: number[];
+    static readonly DefaultTimeProviderInSeconds: () => number;
+    private accounts;
+    constructor(numberOfBuckets?: number, bucketDuration?: number, tiers?: number[], timeProvider?: () => number);
+    logResultToAccount(accountName: string, result: boolean): void;
+    registerStorageAccount(accountName: string): void;
+    getStorageAccount(accountName: string): RankedStorageAccount;
+    getRankedShuffledAccounts(): RankedStorageAccount[];
+}
+//# sourceMappingURL=rankedStorageAccountSet.d.ts.map

package/types/src/resourceManager.d.ts

@@ -1,14 +1,23 @@
 import { Client } from "azure-kusto-data";
+import { TableClient } from "@azure/data-tables";
+export declare enum ResourceType {
+    Queue = 0,
+    Container = 1,
+    Table = 2
+}
 export declare class ResourceURI {
     readonly uri: string;
-    constructor(uri: string);
+    readonly accountName: string;
+    readonly resourceType: ResourceType;
+    constructor(uri: string, accountName: string, resourceType: ResourceType);
 }
 export declare class IngestClientResources {
     readonly securedReadyForAggregationQueues: ResourceURI[] | null;
     readonly failedIngestionsQueues: ResourceURI[] | null;
     readonly successfulIngestionsQueues: ResourceURI[] | null;
     readonly containers: ResourceURI[] | null;
-    constructor(securedReadyForAggregationQueues?: ResourceURI[] | null, failedIngestionsQueues?: ResourceURI[] | null, successfulIngestionsQueues?: ResourceURI[] | null, containers?: ResourceURI[] | null);
+    readonly statusTables: ResourceURI[] | null;
+    constructor(securedReadyForAggregationQueues?: ResourceURI[] | null, failedIngestionsQueues?: ResourceURI[] | null, successfulIngestionsQueues?: ResourceURI[] | null, containers?: ResourceURI[] | null, statusTables?: ResourceURI[] | null);
     valid(): boolean;
 }
 export declare class ResourceManager {
@@ -22,12 +31,17 @@ export declare class ResourceManager {
     authorizationContextLastUpdate: number | null;
     private baseSleepTimeSecs;
     private baseJitterSecs;
+    private rankedStorageAccountSet;
     constructor(kustoClient: Client, isBrowser?: boolean);
     refreshIngestClientResources(): Promise<IngestClientResources>;
     getIngestClientResourcesFromService(): Promise<IngestClientResources>;
     getResourceByName(table: {
         rows: () => any;
-    }, resourceName: string): ResourceURI[];
+    }, resourceName: string, resourceType: ResourceType): ResourceURI[];
+    pupulateStorageAccounts(): void;
+    groupResourcesByStorageAccount(resources: ResourceURI[]): Map<string, ResourceURI[]>;
+    getRankedAndShuffledStorageAccounts(resources: ResourceURI[]): ResourceURI[][];
+    getRoundRobinRankedAndShuffledResources(resources: ResourceURI[]): ResourceURI[];
     refreshAuthorizationContext(): Promise<string>;
     tryRefresh(isAuthContextFetch: boolean): Promise<Error | null>;
     getAuthorizationContextFromService(): Promise<string>;
@@ -36,8 +50,11 @@ export declare class ResourceManager {
     getSuccessfulIngestionsQueues(): Promise<ResourceURI[] | null>;
     getContainers(): Promise<ResourceURI[] | null>;
     getAuthorizationContext(): Promise<string>;
-    getBlockBlobClient(blobName: string): Promise<import("@azure/storage-blob").BlockBlobClient>;
+    getStatusTables(): Promise<ResourceURI[] | null>;
+    createStatusTable(): Promise<TableClient>;
     close(): void;
+    reportResourceUsageResult(accountName: string, success: boolean): void;
 }
+export declare const createStatusTableClient: (uri: string) => TableClient;
 export default ResourceManager;
 //# sourceMappingURL=resourceManager.d.ts.map

package/types/src/streamingIngestClient.browser.d.ts

@@ -4,7 +4,7 @@ import { StreamDescriptor } from "./descriptors";
 import { FileDescriptor } from "./fileDescriptor.browser";
 import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase";
 declare class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
-    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
+    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */

package/types/src/streamingIngestClient.d.ts

@@ -6,7 +6,7 @@ import { StreamDescriptor } from "./descriptors";
 import { FileDescriptor } from "./fileDescriptor";
 import { KustoStreamingIngestClientBase } from "./streamingIngestClientBase";
 declare class KustoStreamingIngestClient extends KustoStreamingIngestClientBase {
-    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
+    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
     /**
      * Use Readable for Node.js and ArrayBuffer in browser
      */

package/types/src/streamingIngestClientBase.d.ts

@@ -4,7 +4,7 @@ import { AbstractKustoClient } from "./abstractKustoClient";
 import { Client as KustoClient, KustoConnectionStringBuilder } from "azure-kusto-data";
 export declare abstract class KustoStreamingIngestClientBase extends AbstractKustoClient {
     protected kustoClient: KustoClient;
-    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput);
+    constructor(kcsb: string | KustoConnectionStringBuilder, defaultProps?: IngestionPropertiesInput, autoCorrectEndpoint?: boolean);
     ingestFromBlob(blob: string | BlobDescriptor, ingestionProperties?: IngestionPropertiesInput, clientRequestId?: string): Promise<any>;
     close(): void;
 }