@neuralinnovations/dataisland-sdk 0.0.1-dev72 → 0.0.1-dev74

package/src/storages/files/file.impl.ts

@@ -1,15 +1,14 @@
 import {Context} from "../../context"
 import {Disposable} from "../../disposable"
 import {
-  FileDto,
+  FileDto, FileProcessingStage,
   FileProgressDto,
   MetadataDto
 } from "../../dto/workspacesResponse"
 import {RpcService} from "../../services/rpcService"
 import {ResponseUtils} from "../../services/responseUtils"
-import {File, FileStatus} from "./file"
+import {File} from "./file"
 import {FilesEvent} from "./files"
-import {isNullOrUndefined} from "../../utils/utils"
 
 export class FileImpl extends File implements Disposable {
   private _isDisposed: boolean = false
@@ -23,8 +22,6 @@ export class FileImpl extends File implements Disposable {
   async initFrom(file: FileDto): Promise<File> {
     this._content = file
 
-    await this.updateStatus()
-
     return this
   }
 
@@ -72,57 +69,24 @@ export class FileImpl extends File implements Disposable {
     return <string>this._content?.previewUrl
   }
 
-  get progress(): FileProgressDto {
-    return <FileProgressDto>this._progress
+  get status(): FileProcessingStage {
+    return <FileProcessingStage>this._content?.stage
   }
 
-  get status(): FileStatus {
-
-    if (
-      isNullOrUndefined(this._progress) || isNullOrUndefined(this._progress.success)
-      || (this._progress.success && this._progress.completed_parts_count < this._progress.file_parts_count)) {
-      return FileStatus.UPLOADING
-    } else if (this._progress.success) {
-      return FileStatus.SUCCESS
-    } else {
-      return FileStatus.FAILED
-    }
+  get progress(): FileProgressDto | undefined {
+    return this._progress
   }
 
-  public fetchAfter() {
-    if (this.status === FileStatus.UPLOADING) {
-      setTimeout(async () => await this.updateStatus(), 2000)
-    }
-  }
+  updateStatus(progress: FileProgressDto): void {
+    if (this._content){
+      this._progress = progress
+      this._content.stage = this._progress.stage
 
-  async updateStatus(): Promise<void> {
-    const response = await this.context
-      .resolve(RpcService)
-      ?.requestBuilder("api/v1/Files/fetch")
-      .searchParam("id", this.id)
-      .sendGet()
-
-    if (ResponseUtils.isFail(response)) {
-      await ResponseUtils.throwError(`Failed to get file ${this.id}`, response)
-    }
-
-    const prev_progress = this._progress
-    this._progress = (await response!.json() as {
-      progress: FileProgressDto
-    }).progress as FileProgressDto
-
-    if (isNullOrUndefined(prev_progress) ||
-      (!isNullOrUndefined(this.progress.success) && this.progress.completed_parts_count > prev_progress.completed_parts_count) ||
-      this.status === FileStatus.SUCCESS ||
-      this.status === FileStatus.FAILED) {
-      // dispatch event, file updated
       this.dispatch({
         type: FilesEvent.UPDATED,
         data: this
       })
     }
-
-    this.fetchAfter()
   }
 
   async update(name: string, metadata: MetadataDto[], description?: string){

package/src/storages/files/file.ts

@@ -1,15 +1,13 @@
-import {FileProgressDto, MetadataDto} from "../../dto/workspacesResponse"
+import {
+  FileProcessingStage,
+  FileProgressDto,
+  MetadataDto
+} from "../../dto/workspacesResponse"
 import { EventDispatcher } from "../../events"
 import { FilesEvent } from "./files"
 
 export type FileId = string
 
-export enum FileStatus {
-  UPLOADING = "uploading",
-  SUCCESS = "success",
-  FAILED = "failed"
-}
-
 /**
  * File.
  */
@@ -50,12 +48,12 @@ export abstract class File extends EventDispatcher<
   /**
    * File uploading progress
    */
-  abstract get progress(): FileProgressDto
+  abstract get progress(): FileProgressDto | undefined
 
   /**
    * File uploading status
    */
-  abstract get status(): FileStatus
+  abstract get status(): FileProcessingStage
 
   /**
    * Get temporary url.
@@ -71,4 +69,10 @@ export abstract class File extends EventDispatcher<
    * Update file.
    */
   abstract update(name: string, metadata: MetadataDto[], description?: string): Promise<void>
+
+  /**
+   * Update file status
+   * @param progress
+   */
+  abstract updateStatus(progress: FileProgressDto): void
 }

package/src/storages/files/files.impl.ts

@@ -1,11 +1,15 @@
 import { Context } from "../../context"
-import { FileDto, FileListResponse } from "../../dto/workspacesResponse"
+import {
+  FileDto,
+  FileListResponse, FileProcessingStage,
+  FilesBatchFetchResponse
+} from "../../dto/workspacesResponse"
 import { RpcService } from "../../services/rpcService"
 import { FileImpl } from "./file.impl"
 import { Files, FilesEvent, UploadFile } from "./files"
 import { WorkspaceImpl } from "../workspaces/workspace.impl"
 import { ResponseUtils } from "../../services/responseUtils"
-import { File } from "./file"
+import {File, FileId} from "./file"
 import { FilesPage } from "./filesPage"
 import { FilesPageImpl } from "./filesPage.impl"
 // import { FormData } from "../../utils/request"
@@ -21,6 +25,8 @@ export class FilesImpl extends Files {
   // Object used as files page data, returned by "query"
   public filesList?: FilesPage
 
+  private _fetchList: FileId[] = []
+
   async upload(files: UploadFile[]): Promise<File[]> {
     const loaded_files = []
     for (const file of files) {
@@ -48,6 +54,35 @@ export class FilesImpl extends Files {
   // INTERNALS
   //----------------------------------------------------------------------------
 
+  async internalFetchQuery(): Promise<void> {
+    const response = await this.context
+      .resolve(RpcService)
+      ?.requestBuilder("api/v1/Files/fetch/batch")
+      .searchParam("organizationId", this.workspace.organization.id)
+      .searchParam("fileIds", this._fetchList.join(","))
+      .sendGet()
+
+    if (ResponseUtils.isFail(response)) {
+      await ResponseUtils.throwError(`Failed to fetch files for org ${this.workspace.organization.id} and workspace ${this.workspace.id}`, response)
+    }
+
+    const progress = (await response!.json()) as FilesBatchFetchResponse
+
+    for (const prg of progress.progress){
+      const file = this.filesList?.files?.find(fl => fl.id == prg.file_id)
+      if (file) {
+        file.updateStatus(prg)
+        if (file.status > FileProcessingStage.PROCESSING){
+          const index = this._fetchList.indexOf(file.id)
+          this._fetchList.splice(index, 1)
+        }
+      }
+    }
+
+    if (this._fetchList.length != 0){
+      setTimeout(async () => await this.internalFetchQuery(), 2000)
+    }
+  }
 
   async internalGetFile(id: string): Promise<File>{
     if (id === undefined || id === null) {
@@ -163,6 +198,8 @@ export class FilesImpl extends Files {
       )
     }
 
+    this._fetchList = []
+
     // parse files from the server's response
     const files = (await response!.json()) as FileListResponse
 
@@ -180,11 +217,19 @@ export class FilesImpl extends Files {
 
       // add file to the collection
       filesList.files.push(file)
+
+      if (file.status <= FileProcessingStage.PROCESSING) {
+        this._fetchList.push(file.id)
+      }
     }
 
     // set files list
     this.filesList = filesList
 
+    if (this._fetchList.length != 0){
+      setTimeout(async () => await this.internalFetchQuery(), 2000)
+    }
+
     return filesList
   }
 

package/src/storages/organizations/organization.impl.ts

@@ -36,6 +36,8 @@ import {
 } from "../../dto/apiKeyResponse"
 import {IconResponse} from "../../dto/workspacesResponse"
 import {UploadFile} from "../files/files"
+import {QueryFlowsImpl} from "../queryFlows/queryFlows.impl"
+import {QueryFlows} from "../queryFlows/queryFlows"
 
 export class OrganizationImpl extends Organization implements Disposable {
   private _isDisposed: boolean = false
@@ -43,6 +45,7 @@ export class OrganizationImpl extends Organization implements Disposable {
   private _content?: OrganizationDto
   private readonly _workspaces: WorkspacesImpl
   private readonly _accessGroups: GroupsImpl
+  private readonly _queryFlows: QueryFlowsImpl
   private readonly _chats: ChatsImpl
 
   constructor(private readonly context: Context) {
@@ -50,6 +53,7 @@ export class OrganizationImpl extends Organization implements Disposable {
     this._workspaces = new WorkspacesImpl(this, this.context)
     this._accessGroups = new GroupsImpl(this, this.context)
     this._chats = new ChatsImpl(this, this.context)
+    this._queryFlows = new QueryFlowsImpl(this, this.context)
   }
 
   public async initFrom(
@@ -63,7 +67,8 @@ export class OrganizationImpl extends Organization implements Disposable {
     const promises = [
       this._workspaces.initFrom(content.id),
       this._chats.initFrom(content.id),
-      this._accessGroups.initialize()
+      this._accessGroups.initialize(),
+      this._queryFlows.init()
     ]
 
     await Promise.all(promises)
@@ -107,6 +112,10 @@ export class OrganizationImpl extends Organization implements Disposable {
     return this._accessGroups
   }
 
+  get queryFlows(): QueryFlows {
+    return this._queryFlows
+  }
+
   get chats(): Chats {
     return this._chats
   }

package/src/storages/organizations/organization.ts

@@ -16,6 +16,7 @@ import {QuizData} from "../../dto/quizResponse"
 import {InviteResponse} from "../../dto/invitesResponse"
 import { OrganizationApiKey } from "../../dto/apiKeyResponse"
 import {UploadFile} from "../files/files"
+import {QueryFlows} from "../queryFlows/queryFlows"
 
 /**
  * Organization event.
@@ -66,6 +67,11 @@ export abstract class Organization extends EventDispatcher<
    */
   abstract get accessGroups(): Groups
 
+  /**
+   * Query flows
+   */
+  abstract get queryFlows(): QueryFlows
+
   /**
    * Get organization members
    */

package/src/storages/queryFlows/queryFlow.impl.ts

@@ -0,0 +1,74 @@
+import {QueryFlow, QueryFlowEvent} from "./queryFlow"
+import {QueryFlowDto, QueryFlowStatus} from "../../dto/queryFlowResponse"
+import {FlowId} from "./queryFlows"
+import {Context} from "../../context"
+import {RpcService} from "../../services/rpcService"
+import {ResponseUtils} from "../../services/responseUtils"
+
+
+export class QueryFlowImpl extends QueryFlow {
+  private _content?: QueryFlowDto
+  private _id?: FlowId
+
+  constructor(private readonly context: Context) {
+    super()
+  }
+
+  async initFrom(id: FlowId) {
+    this._id = id
+    await this.fetch()
+  }
+
+  async fetch() {
+    const response = await this.context
+      .resolve(RpcService)
+      ?.requestBuilder("api/v1/QueryFlows")
+      .searchParam("flowId", this.id)
+      .sendGet()
+
+    // check response status
+    if (ResponseUtils.isFail(response)) {
+      await ResponseUtils.throwError(`Failed to fetch query flow with id ${this.id}`, response)
+    }
+
+    // parse flow from the server's response
+    const flow = (await response!.json()) as QueryFlowDto
+
+    const lastState = this._content?.state
+
+    this._content = flow
+    if (lastState !== undefined && lastState !== this._content.state){
+      this.dispatch({
+        type: QueryFlowEvent.UPDATED,
+        data: this
+      })
+    }
+
+    if (this._content.state === QueryFlowStatus.IN_PROGRESS) {
+      setTimeout(async () => await this.fetch(), 2000)
+    }
+  }
+
+
+  get id(): FlowId {
+    if (this._id)
+      return this._id
+    else
+      throw new Error("Query fLow is not loaded")
+  }
+
+  get resultUrl(): string {
+    if (this._content && this._content.result.fileUrl)
+      return this._content.result.fileUrl
+    else
+      throw new Error("Query fLow is not loaded")
+  }
+
+  get status(): QueryFlowStatus {
+    if (this._content)
+      return this._content.state
+    else
+      throw new Error("Query fLow is not loaded")
+  }
+
+}

package/src/storages/queryFlows/queryFlow.ts

@@ -0,0 +1,20 @@
+import {EventDispatcher} from "../../events"
+import {FlowId} from "./queryFlows"
+import {QueryFlowStatus} from "../../dto/queryFlowResponse"
+
+export enum QueryFlowEvent {
+  UPDATED = "updated"
+}
+
+export abstract class QueryFlow extends EventDispatcher<
+  QueryFlowEvent,
+  QueryFlow
+> {
+
+  abstract get id(): FlowId
+
+  abstract get status(): QueryFlowStatus
+
+  abstract get resultUrl(): string
+
+}

package/src/storages/queryFlows/queryFlows.impl.ts

@@ -0,0 +1,151 @@
+
+import {Context} from "../../context"
+import {FlowId, QueryFlows, QueryFlowsEvent} from "./queryFlows"
+import {QueryFlow} from "./queryFlow"
+import {OrganizationImpl} from "../organizations/organization.impl"
+import {RpcService} from "../../services/rpcService"
+import {ResponseUtils} from "../../services/responseUtils"
+import {WorkspaceId} from "../workspaces/workspaces"
+import {
+  QueryFlowListResponse,
+  QueryFlowResponse
+} from "../../dto/queryFlowResponse"
+import {QueryFlowImpl} from "./queryFlow.impl"
+import {FileId} from "../files/file"
+import {UploadFile} from "../files/files"
+
+
+export class QueryFlowsImpl extends QueryFlows {
+
+  private _collection: QueryFlow[] = []
+
+  constructor(
+    public readonly organization: OrganizationImpl,
+    public readonly context: Context) {
+    super()
+  }
+
+  async init() {
+    const response = await this.context
+      .resolve(RpcService)
+      ?.requestBuilder("api/v1/QueryFlows/list")
+      .searchParam("organizationId", this.organization.id)
+      .sendGet()
+
+    // check response status
+    if (ResponseUtils.isFail(response)) {
+      await ResponseUtils.throwError(`Failed to fetch query flows for org ${this.organization.id}`, response)
+    }
+
+    // parse flows from the server's response
+    const flows = ((await response!.json()) as QueryFlowListResponse)
+      .flowIds
+
+    // init flows from the server's response
+    for (const flow of flows) {
+      // create workspace implementation
+      const flowImpl = new QueryFlowImpl(this.context)
+
+      // init workspace from the server's response
+      await flowImpl.initFrom(flow)
+
+      // add workspace to the collection
+      this._collection.push(flowImpl)
+
+      // dispatch event
+      this.dispatch({
+        type: QueryFlowsEvent.ADDED,
+        data: flowImpl
+      })
+    }
+  }
+
+  get collection(): QueryFlow[] {
+    return this._collection
+  }
+
+  async create(workspaceId: WorkspaceId, fileId: FileId, file: UploadFile ): Promise<FlowId> {
+    if (workspaceId === undefined || workspaceId === null || workspaceId.trim() === "") {
+      throw new Error("WorkspaceId is required, must be not empty")
+    }
+    if (fileId === undefined || fileId === null || fileId.trim() === "") {
+      throw new Error("FileId is required, must be not empty")
+    }
+    if (file === undefined || file === null) {
+      throw new Error("Create query flow, file is undefined or null")
+    }
+
+    // form data to send
+    const form = new FormData()
+    form.append("organizationId", this.organization.id)
+    form.append("workspaceId", workspaceId)
+    form.append("fileId", fileId)
+    form.append("file", file, file.name)
+
+    // send request to the server
+    const response = await this.context
+      .resolve(RpcService)
+      ?.requestBuilder("api/v1/QueryFlows")
+      .sendPostFormData(form)
+
+    // check response status
+    if (ResponseUtils.isFail(response)) {
+
+      await ResponseUtils.throwError(`Query flow creation for ${file.name}`, response)
+    }
+
+    const content = (await response!.json()) as QueryFlowResponse
+    // create workspace implementation
+    const queryFlow = new QueryFlowImpl(this.context)
+    await queryFlow.initFrom(content.flowId)
+
+    // add workspace to the collection
+    this._collection.push(queryFlow)
+
+
+    // dispatch event
+    this.dispatch({
+      type: QueryFlowsEvent.ADDED,
+      data: queryFlow
+    })
+
+    return queryFlow.id
+  }
+
+  async delete(id: FlowId): Promise<void> {
+    const flow = this._collection.find(flow => flow.id == id)
+
+    if (!flow) {
+      throw new Error(`Query flow ${id} is not found`)
+    }
+
+    // send delete request to the server
+    const response = await this.context
+      .resolve(RpcService)
+      ?.requestBuilder("api/v1/QueryFlows")
+      .searchParam("flowId", id)
+      .sendDelete()
+
+    // check response status
+    if (ResponseUtils.isFail(response)) {
+      await ResponseUtils.throwError(
+        `Failed to delete query flow ${id} in organization: ${this.organization.id}`,
+        response
+      )
+    }
+
+    // remove query flow from the collection
+    const index = this._collection.indexOf(flow)
+    if (index < 0) {
+      throw new Error(`Query flow ${id} is not found`)
+    }
+    this._collection.splice(index, 1)
+
+    // dispatch event
+    this.dispatch({
+      type: QueryFlowsEvent.REMOVED,
+      data: flow
+    })
+  }
+
+}

package/src/storages/queryFlows/queryFlows.ts

@@ -0,0 +1,32 @@
+import {EventDispatcher} from "../../events"
+import {QueryFlow} from "./queryFlow"
+import {WorkspaceId} from "../workspaces/workspaces"
+import {FileId} from "../files/file"
+import {UploadFile} from "../files/files"
+
+
+/**
+ * Query flow id.
+ */
+export type FlowId = string
+
+/**
+ * Query flow event.
+ */
+export enum QueryFlowsEvent {
+  ADDED = "added",
+  REMOVED = "removed"
+}
+
+export abstract class QueryFlows extends EventDispatcher<
+  QueryFlowsEvent,
+  QueryFlow
+> {
+
+  abstract get collection(): QueryFlow[]
+
+  abstract create(workspaceId: WorkspaceId, fileId: FileId, file: UploadFile ): Promise<FlowId>
+
+  abstract delete(id: FlowId): Promise<void>
+
+}