@whitewall/blip-sdk 0.0.138 → 0.0.139

package/src/namespaces/account.ts

@@ -17,7 +17,7 @@ import type {
     ThreadItem,
 } from '../types/index.ts'
 import type { Notification } from '../types/notification.ts'
-import type { ODataFilter } from '../utils/odata.ts'
+import { type ODataFilter, filter } from '../utils/odata.ts'
 import { uri } from '../utils/uri.ts'
 import { type ConsumeOptions, Namespace, type SendCommandOptions } from './namespace.ts'
 
@@ -217,6 +217,59 @@ export class AccountNamespace extends Namespace {
         )
     }
 
+    public async *streamContactsInTimeframe(
+        startDate: Date | string,
+        endDate: Date | string,
+        opts: Omit<ConsumeOptions, 'skip' | 'max'> = {},
+    ): AsyncIterable<Contact> {
+        const take = opts.take ?? 1000
+        const start = new Date(startDate)
+        const end = new Date(endDate)
+        if (start > end) return
+
+        const seenIdentities = new Set<string>()
+        let cursorTime = end.getTime()
+        const startTime = start.getTime()
+
+        while (cursorTime >= startTime) {
+            const items = await this.sendCommand<'get', Array<Contact>>(
+                {
+                    method: 'get',
+                    uri: uri`/contacts?${{
+                        $filter: filter<Contact>()
+                            .ge('lastMessageDate', start)
+                            .le('lastMessageDate', new Date(cursorTime)),
+                    }}`,
+                },
+                { collection: true, ...opts, take, fetchall: false, optimistic: false },
+            )
+
+            let oldestTime = Number.POSITIVE_INFINITY
+
+            for (const contact of items) {
+                if (contact.lastMessageDate) {
+                    if (!seenIdentities.has(contact.identity)) {
+                        seenIdentities.add(contact.identity)
+                        yield contact
+                    }
+
+                    const lastMessageTime = new Date(contact.lastMessageDate).getTime()
+                    if (lastMessageTime < oldestTime) {
+                        oldestTime = lastMessageTime
+                    }
+                }
+            }
+
+            if (items.length < take || !opts.fetchall) break
+
+            if (oldestTime < Number.POSITIVE_INFINITY) {
+                cursorTime = oldestTime - 1
+            } else {
+                break
+            }
+        }
+    }
+
     public async getContactComments(contact: Identity, opts?: ConsumeOptions): Promise<Array<Comment>> {
         try {
             return await this.sendCommand(
@@ -401,6 +454,7 @@ export class AccountNamespace extends Namespace {
         )
     }
 
+    // THis doesn't ensure that the contact exists, it may have received a active message but never interacted
     public async getThreads(startDate: Date | string, endDate: Date | string, opts: Omit<ConsumeOptions, 'skip'> = {}) {
         const start = new Date(startDate)
         const end = new Date(endDate)

package/src/namespaces/namespace.ts

@@ -4,6 +4,12 @@ import { type Command, type CommandMethods, type Domain, type Identity, Node, ty
 import { randomId } from '../utils/random.ts'
 import { type URI, uriToString } from '../utils/uri.ts'
 
+type Unpacked<T> = T extends Array<infer U> ? U : T
+type PromiseOrIterator<T, S extends boolean> = S extends true ? AsyncIterable<Unpacked<T>> : Promise<T>
+type NamespaceCommand<TMethod extends CommandMethods = CommandMethods> = Omit<Command<TMethod>, 'uri' | 'id' | 'to'> & {
+    uri: URI
+}
+
 export type SendCommandOptions = {
     to?: NodeLike
     ownerIdentity?: Identity | Domain
@@ -15,8 +21,11 @@ export type ConsumeOptions = SendCommandOptions & {
     skip?: number
     fetchall?: boolean
     max?: number
-    // If true, it will try to parallel fetch data in chunks
-    // Be careful as this can be slower in situations with few data and will consume much more bandwidth
+    /**
+     *  If true, it will try to parallel fetch data in chunks
+     *  Be careful as this can be slower in situations with few data and will consume much more bandwidth
+     *  Set together with max or fetchall
+     */
     optimistic?: boolean
 }
 
@@ -31,93 +40,159 @@ export class Namespace {
         return `postmaster@${this.ownerSubdomain ? `${this.ownerSubdomain}.` : ''}${this.defaultOptions?.domain ?? 'msging.net'}` as Identity
     }
 
-    public async sendCommand<
-        TMethod extends CommandMethods,
-        TResponse = void,
-        TCollection = TResponse extends Array<unknown> ? true : false,
-    >(
-        command: Omit<Command<TMethod>, 'uri' | 'id' | 'to'> & {
-            uri: URI
-        },
-        opts?: ConsumeOptions & {
-            collection?: TCollection extends true ? true : never
-        },
-    ): Promise<TResponse> {
+    public sendCommand<TMethod extends CommandMethods, TResponse = void>(
+        command: NamespaceCommand<TMethod>,
+        opts: ConsumeOptions & { collection: true; stream: true },
+    ): PromiseOrIterator<TResponse, true>
+    public sendCommand<TMethod extends CommandMethods, TResponse = void>(
+        command: NamespaceCommand<TMethod>,
+        opts?: ConsumeOptions & { collection?: boolean; stream?: false },
+    ): PromiseOrIterator<TResponse, false>
+    public sendCommand<TMethod extends CommandMethods, TResponse = void>(
+        command: NamespaceCommand<TMethod>,
+        opts?: ConsumeOptions & { collection?: boolean; stream?: boolean },
+    ): Promise<TResponse> | AsyncIterable<Unpacked<TResponse>> {
         const options = { ...this.defaultOptions, ...opts }
 
-        let path = command.uri.path
-        const query = command.uri.query
+        if (options.stream) {
+            return this.sendIterableCommand(command, options)
+        } else if (options.collection) {
+            return this.sendCollectionCommand(command, options)
+        } else {
+            return this.sendPlainCommand(command, options)
+        }
+    }
 
-        const take = options?.take ?? Math.min(options?.max ?? 100, 100)
+    private prepareCommandContext<TMethod extends CommandMethods>(
+        command: NamespaceCommand<TMethod>,
+        options: ConsumeOptions,
+    ) {
+        const domain =
+            options?.domain ??
+            (this.blipClient.sender instanceof ConnectionSender ? this.blipClient.sender.domain : 'msging.net')
 
-        if (options?.collection) {
-            query.set('$take', take.toString())
+        return {
+            path: options?.ownerIdentity ? `lime://${options.ownerIdentity}${command.uri.path}` : command.uri.path,
+            query: new Map(command.uri.query),
+            owner: new Node('postmaster', `${this.ownerSubdomain ? `${this.ownerSubdomain}.` : ''}${domain}`),
+            take: options?.take ?? Math.min(options?.max ?? 100, 100),
         }
+    }
 
-        if (options?.ownerIdentity) {
-            path = `lime://${options.ownerIdentity}${path}`
+    private buildCommand<TMethod extends CommandMethods>(
+        base: NamespaceCommand<TMethod>,
+        path: string,
+        query: Map<string, string>,
+        owner: Node,
+        options: ConsumeOptions,
+    ): Command<TMethod> {
+        return {
+            id: randomId(),
+            to: options.to ?? owner,
+            ...base,
+            uri: uriToString({ path, query }),
+        } as Command<TMethod>
+    }
+
+    private extractCollection<T>(response: unknown): Array<T> {
+        if (response && typeof response === 'object') {
+            const r = response as Record<string, unknown>
+            if (Array.isArray(r.items)) return r.items as Array<T>
+            if (Array.isArray(r.data)) return r.data as Array<T>
         }
+        return []
+    }
 
-        const domain =
-            options?.domain ??
-            (this.blipClient.sender instanceof ConnectionSender ? this.blipClient.sender.domain : 'msging.net')
-        const owner = new Node('postmaster', `${this.ownerSubdomain ? `${this.ownerSubdomain}.` : ''}${domain}`)
-
-        if (!options.collection) {
-            const commandToSend = {
-                id: randomId(),
-                to: options.to ?? owner,
-                ...command,
-                uri: uriToString({ path, query }),
-            } as Command<TMethod>
+    private async *sendIterableCommand<TMethod extends CommandMethods, TResponse = void>(
+        command: Omit<Command<TMethod>, 'uri' | 'id' | 'to'> & {
+            uri: URI
+        },
+        options: ConsumeOptions,
+    ): AsyncIterable<Unpacked<TResponse>> {
+        const { path, query, owner, take } = this.prepareCommandContext(command, options)
+        let skip = options.skip ?? 0
+        let yielded = 0
+
+        while (true) {
+            const q = new Map(query).set('$take', take.toString()).set('$skip', skip.toString())
+
+            const commandToSend = this.buildCommand(command, path, q, owner, options)
             const response = await this.blipClient.sender.sendCommand(commandToSend)
-            return response as TResponse
+            const items = this.extractCollection<Unpacked<TResponse>>(response)
+
+            if (!items.length) {
+                break
+            }
+
+            for (const item of items) {
+                yield item
+
+                if (options.max && yielded++ >= options.max) {
+                    return
+                }
+            }
+
+            if (!options.fetchall || items.length < take) {
+                break
+            }
+
+            skip += take
         }
+    }
 
+    private async sendPlainCommand<TMethod extends CommandMethods, TResponse = void>(
+        command: Omit<Command<TMethod>, 'uri' | 'id' | 'to'> & {
+            uri: URI
+        },
+        options: ConsumeOptions,
+    ): Promise<TResponse> {
+        const { path, query, owner } = this.prepareCommandContext(command, options)
+
+        const commandToSend = this.buildCommand(command, path, query, owner, options)
+        const response = await this.blipClient.sender.sendCommand(commandToSend)
+        return response as TResponse
+    }
+
+    private async sendCollectionCommand<TMethod extends CommandMethods, TResponse = void>(
+        command: Omit<Command<TMethod>, 'uri' | 'id' | 'to'> & {
+            uri: URI
+        },
+        options: ConsumeOptions,
+    ): Promise<TResponse> {
+        const { path, query, owner, take } = this.prepareCommandContext(command, options)
         if (options.optimistic && !options.max && !options.fetchall) {
             throw new Error('Optimistic consume requires max or fetchall to be set')
         }
 
         let skip = options.skip ?? 0
-        let results: Array<TResponse> = []
+        const results: Array<Unpacked<TResponse>> = []
 
+        query.set('$take', take.toString())
         while (true) {
             const remaining = options.max !== undefined ? options.max - results.length : Number.POSITIVE_INFINITY
-            const potentialBatches = Math.ceil(remaining / take)
-            const batchSize = options.optimistic ? Math.min(potentialBatches, 20) : 1
-            const tasks = Array.from({ length: batchSize }, async (_, i: number) => {
-                const currentQuery = new Map(query)
-                currentQuery.set('$skip', (skip + i * take).toString())
-
-                const commandToSend = {
-                    id: randomId(),
-                    to: options.to ?? owner,
-                    ...command,
-                    uri: uriToString({ path, query: currentQuery }),
-                } as Command<TMethod>
-
-                const response = (await this.blipClient.sender.sendCommand(commandToSend)) as
-                    | { data: Array<TResponse> }
-                    | { items: Array<TResponse> }
-                const items: Array<TResponse> =
-                    'items' in response ? response.items : 'data' in response ? response.data : []
-
-                return items ?? []
-            })
-
-            const items = await Promise.all(tasks)
-            results = results.concat(items.flat())
+            const pageCapacity = options.optimistic ? Math.min(Math.ceil(remaining / take), 20) : 1
+
+            const items = await Promise.all(
+                Array.from({ length: pageCapacity }, async (_, i: number) => {
+                    const q = new Map(query).set('$skip', (skip + i * take).toString())
+                    const cmd = this.buildCommand(command, path, q, owner, options)
+                    const response = await this.blipClient.sender.sendCommand(cmd)
+                    return this.extractCollection<Unpacked<TResponse>>(response)
+                }),
+            )
+
+            const flattened = items.flat()
+            results.push(...flattened)
 
             if (
                 !options.fetchall ||
-                results.length % take !== 0 ||
-                results.length === 0 ||
+                flattened.length < take ||
                 (options.max !== undefined && results.length >= options.max)
             ) {
                 break
             }
 
-            skip += take * batchSize
+            skip += take * pageCapacity
         }
 
         return results as TResponse

package/src/namespaces/portal.ts

@@ -297,7 +297,10 @@ export class PortalNamespace extends Namespace {
                 method: 'get',
                 uri: uri`/applications/${Node.from(bot, 'msging.net')}/users`,
             },
-            opts,
+            {
+                collection: true,
+                ...opts,
+            },
         )
     }
 

package/src/sender/enveloperesolver.ts

@@ -114,9 +114,9 @@ export class EnvelopeResolver {
     }
 
     public createEnvelopeResponsePromise(id: string): Promise<Envelope> {
-        return new Promise((resolve) => {
-            this.waitingEnvelopeResponseResolvers[id] = resolve
-        })
+        const { resolve, promise } = Promise.withResolvers<Envelope>()
+        this.waitingEnvelopeResponseResolvers[id] = resolve
+        return promise
     }
 
     public addListener<K extends keyof EventMap>(ev: K, listener: Listener<K>) {

package/src/sender/throttler.ts

@@ -1,36 +1,75 @@
 export class EnvelopeThrottler {
-    private sent = {
+    private buckets = {
         message: {
             max: 50,
-            started: 0,
-            count: 0,
+            tokens: 0,
+            lastRefill: 0,
+            waiting: [] as Array<() => void>,
+            timer: null as NodeJS.Timeout | null,
         },
         command: {
             max: 200,
-            started: 0,
-            count: 0,
+            tokens: 0,
+            lastRefill: 0,
+            waiting: [] as Array<() => void>,
+            timer: null as NodeJS.Timeout | null,
         },
     }
 
-    public async throttle(type: keyof typeof this.sent): Promise<void> {
-        const throughput = this.sent[type].max * 0.9
-        const timeToReset = 1000
+    private readonly refillInterval = 1000 // ms
+    private readonly throughputFactor = 0.9
+    // memo for fast-path (zero allocation)
+    private readonly voidPromise = Promise.resolve()
 
-        while (true) {
-            const elapsed = Date.now() - this.sent[type].started
+    public throttle(type: keyof typeof this.buckets): Promise<void> {
+        const bucket = this.buckets[type]
+        this.refillTokens(type)
 
-            if (elapsed >= timeToReset) {
-                this.sent[type].started = Date.now()
-                this.sent[type].count = 0
+        if (bucket.tokens > 0) {
+            bucket.tokens--
+            return this.voidPromise
+        }
+
+        const { resolve, promise } = Promise.withResolvers<void>()
+        bucket.waiting.push(resolve)
+
+        this.scheduleRefill(type)
+
+        return promise
+    }
+
+    private refillTokens(type: keyof typeof this.buckets): void {
+        const bucket = this.buckets[type]
+        const now = Date.now()
+        const elapsed = now - bucket.lastRefill
+
+        if (elapsed >= this.refillInterval) {
+            bucket.tokens = Math.floor(bucket.max * this.throughputFactor)
+            bucket.lastRefill = now
+        }
+    }
+
+    private scheduleRefill(type: keyof typeof this.buckets): void {
+        const bucket = this.buckets[type]
+        if (bucket.timer) return
+
+        const delay = Math.max(0, this.refillInterval - (Date.now() - bucket.lastRefill))
+
+        bucket.timer = setTimeout(() => {
+            bucket.timer = null
+            this.refillTokens(type)
+
+            while (bucket.tokens > 0 && bucket.waiting.length > 0) {
+                const resolve = bucket.waiting.shift()
+                if (resolve) {
+                    bucket.tokens--
+                    resolve()
+                }
             }
 
-            if (this.sent[type].count < throughput) {
-                this.sent[type].count++
-                break
-            } else {
-                const wait = timeToReset - elapsed
-                await new Promise((resolve) => setTimeout(resolve, wait))
+            if (bucket.waiting.length > 0) {
+                this.scheduleRefill(type)
             }
-        }
+        }, delay)
     }
 }