corebasic 1.0.230 → 1.0.232

package/libs/features.ts

@@ -8,11 +8,90 @@ import axios from 'axios'
 // @ts-ignore
 import jwt from 'jsonwebtoken'
 
-export type Message = unknown
-export type Req = unknown
-export type Res = unknown
+
+// ===================================
+
+type HttpMethod = "get" | "post" | "put" | "delete" | "patch" | "head" | "options" | "connect" | "trace"
+
+
+type FeatureMeta = {
+    feature: string,
+    app: string,
+    user: string,
+    outlet: string,
+    company: string,
+    staff: string,
+    client: string,
+    version: string,
+    txn: string,
+    // invoiceTxn?: string // TODO: Must also include invoiceTxn
+}
+type FeatureParams = Record<string, string>
+type FeatureMessage = {
+    meta: FeatureMeta
+    data: unknown
+    params: FeatureParams
+    // To be removed 1
+    date: number // used as message.date
+    // To be removed 2
+    topic: string // used as req.body.topic
+    feature: string // used as req.body.feature or message.feature
+    user: string    // used as req.body.user or message.user
+    txn: string     // used as message.txn
+
+
+    // client  // Only used as req.body.client in auth.ts, session.ts and messaging.ts
+    // version // Only used as req.body.version in auth.ts and session.ts
+    // outlet, company // Only used as req.body.outlet, req.body.company in session.ts
+    // app // Only used as req.body.app in auth.ts and session.ts
+}
+type PayloadMessage = FeatureMeta & {
+    data: unknown
+    params?: FeatureParams
+}
+
+// =========
+
+export type Req = {
+    meta: FeatureMeta
+    body: FeatureMessage
+    params: FeatureParams
+    method: string
+    path: string
+    url?: string
+    headers?: Record<string, string | string[] | undefined>
+    query?: Record<string, unknown>
+    on?: (event: string, callback: () => void) => void
+}
+
+type ResJson = (response: unknown) => void
+type ResSend = (response: unknown) => void;
+type ResStatus = (status: number) => {send: ResSend, json: ResJson};
+export type Res = {
+    send: ResSend
+    json: ResJson
+    end: (finalData?: string) => void
+    status: ResStatus
+    sendStatus: (status: number) => void
+    setHeader: (header: string, value: string) => void
+    flushHeaders: () => void
+    headersSent: boolean
+    writableEnded: boolean
+}
+
+
+// ========= Express Types ==========
+type ExpressRequest = Omit<Req, "meta"  | "body"> & { body: PayloadMessage }
+type ExpressResponse = Res
+type ExpressHandler = (req: ExpressRequest, res: ExpressResponse) => Promise<void>
+type ExpressApplication = { [Method in HttpMethod]: (path: string, handler: ExpressHandler) => unknown }
+// ===================================
+
+
+// export type Req = unknown
+// export type Res = unknown
 export type Query = (req: Req, res: Res) => Promise<void>
-export type Command = (topic: string, message: Message, req?: Req, res?: Res) => Promise<void>
+export type Command = (topic: string, message: FeatureMessage, req?: Req, res?: Res) => Promise<void>
 type Handler = Query | Command
 
 
@@ -28,11 +107,13 @@ type FeatureEntry = { api: string, topic: string, bypass: boolean, featureless:
 
 
 let features: Record<string, FeatureEntry> = {}
-let apis = {}
+let apis: Record<string, Record<string, FeatureEntry>> = {
+    // <api>: {<feature>: FeatureEntry}
+}
 
-const getFeatureMethod = api => api.split(' ')[0].toLowerCase()
-const getFeatureUrl = api => api.split(' ')[1]
-const getFeature = name => features[name]
+const getFeatureMethod = (api: string): HttpMethod => api.split(' ')[0]!.toLowerCase() as HttpMethod
+const getFeatureUrl = (api: string): string => { const url = api.split(' ')[1]; if (!url) throw new Error(`Invalid Feature url in api: ${api}`); return url}
+const getFeature = (name: string): FeatureEntry | undefined => features[name]
 
 const DEPLOY_TOKEN_SECRET = process.env.DEPLOY_TOKEN_SECRET || "MY_SECRET_DEPLOY_TOKEN"
 const SERVICE_ACCESS_TOKEN = jwt.sign({app: process.env.APP_DEPLOYMENT_NAME}, DEPLOY_TOKEN_SECRET, { expiresIn: '365d' });
@@ -63,39 +144,50 @@ let appId = Utils.uid()
 let SERVICE_ADDRESS = process.env.APP_ENDPOINT || 'http://127.0.0.1:3000'
 
 
-export const get = feature => {
+export const get = (feature: string) => {
     const throwError = () => {throw new Error(`Feature ${feature} not found in internal or external list during Features.subscribe call`)}
-    const baseFeature: FeatureEntry = getFeature(feature)
+    const baseFeature: FeatureEntry | undefined = getFeature(feature)
     let {api, service = SERVICE_ADDRESS, topic} = baseFeature ? baseFeature as FeatureEntry & {service: string} : (SLYP_FEATURES_LIST[feature] ?? throwError())
     return {api, service, topic}
 }
 
-export const send = async (meta, feature, data, params) => {
+export const send = async (meta: Partial<FeatureMeta>, feature: string, data?: unknown, params?: FeatureParams) => {
     const throwError = () => {throw new Error(`Feature ${feature} not found in internal or external list during inter feature call`)}
-    const baseFeature: FeatureEntry = getFeature(feature)
+    const baseFeature: FeatureEntry | undefined = getFeature(feature)
     let {api, service = SERVICE_ADDRESS} = baseFeature ? baseFeature as FeatureEntry & {service: string} : (SLYP_FEATURES_LIST[feature] ?? throwError())
     let method = getFeatureMethod(api)
     let url = getFeatureUrl(api)
 
     if (params || url.includes(':')) {
         for (let key in params)
-            url = url.replace(`:${key}`, params[key])
+            url = url.replace(`:${key}`, params[key]!)
         if (url.includes(':'))
             throw new Error(`Error: Internal feature call send on feature: ${feature} when api/params/substitution`)
     }
 
-    let payload = { ...meta, data, feature, txn: Utils.uid() }
+    const payload: PayloadMessage = { ...meta, data, feature, txn: Utils.uid() } as PayloadMessage
     service = service.replace('http://slyp.app', 'https://slyp.app')
 
     // return  (await axios[method](`${service}${url}`, { data: payload, headers: {jwt: headers.jwt}, timeout: 1000 })).data
 
     // Used to work. But request to 127.0.0.1 fails to send the specified headers along with the request.
     // return  (await axios[method](`${service}${url}`, payload, {headers: {jwt: SERVICE_ACCESS_TOKEN, service: true}, timeout: 1000 })).data // Worked Earlier, but issue spotted
-    if (getFeature(feature)) { // Local call
+    if (baseFeature) { // Local call
         let response;
-        let req = {meta,body: payload, params,method}
+        let req: ExpressRequest = { body: payload, params: params ?? {}, method, path: url, url: ''}
         req = JSON.parse(JSON.stringify(req))
-        let res = {json: payload => response = payload, end: () => true  }
+        const callback = (payload: unknown): void => { response = payload }
+        const res: ExpressResponse = {
+            json: callback,
+            send: callback,
+            end: () => {},
+            status: (_status: number) => { return {send: callback, json: callback} },
+            sendStatus: (_status: number) => {},
+            setHeader: (_header: string, _value: string) => {},
+            flushHeaders: () => {},
+            headersSent: false,
+            writableEnded: false,
+        }
         await apiHandler(req, res);
         return response
     }
@@ -109,7 +201,7 @@ export const send = async (meta, feature, data, params) => {
       })).data;
 }
 
-let appids = {}
+let appids: Record<string, boolean> = {}
 async function announce() {
     let exp_features: Record<string, RemoteFeatureEntry> = {}
     for (let [key, {api,subscribe}] of Object.entries(features))
@@ -128,17 +220,18 @@ async function announce() {
 }
 
 
-function getFeaturelessFeature(req) {
+function getFeaturelessFeature(req: ExpressRequest) {
     if (Session.ALLOWED_URLS.includes(req.path)) {
-        for(let key in features){
-            if (features[key].featureless && features[key].api === req.method.toUpperCase() + ' ' + req.path) {
+        for(let key in features) {
+            const feature = features[key]!
+            if (feature.featureless && feature.api === req.method.toUpperCase() + ' ' + req.path) {
                 return getFeature(key)
             }
         }
     }
 }
 
-const apiHandler = async (req, res) => {
+const apiHandler = async (req: ExpressRequest, res: ExpressResponse) => {
     let method = req.method.toLowerCase()
     let featureless
     try {
@@ -155,50 +248,89 @@ const apiHandler = async (req, res) => {
             throw { status: 404, message: `Resource not found. Feature ${req.body.feature} not available.` }
         }
 
+        feature = feature! // typescript validation with !
+
         let topic = feature.topic
-        let params = getFeatureUrl(feature.api).split("/").filter(item => item.startsWith(":")).map(item => item.replace(":", ""))
+        let params = getFeatureUrl(feature.api).split("/").filter((item: string) => item.startsWith(":")).map((item: string) => item.replace(":", ""))
         for (let param of params)
             if (!req.params[param])
                 throw { status: 404, message: "Resource not found. One or more url parameter not specified." }
 
-        let meta = {...req.body, data: undefined}
-        req.meta = meta
+        let meta = {...req.body, data: undefined} as FeatureMeta // TODO: Must also include invoiceTxn
+
         if (process.env.USE_DEFAULT_COMPANY) {
-            req.meta.company = 'DEFAULT_COMPANY'
-            req.meta.outlet = 'DEFAULT_OUTLET'
+            meta.company = 'DEFAULT_COMPANY'
+            meta.outlet = 'DEFAULT_OUTLET'
         }
 
+        meta.txn = meta.txn ?? Utils.uid()
+
+        const message: FeatureMessage = {
+            meta,
+            data: req.body.data,
+            params: req.params,
+
+            // To be removed 1
+            date: new Date().getTime(),
+
+            // To be removed 2
+            topic,
+            feature: req.body.feature,
+            user: req.body.user,
+            txn: meta.txn,
+        }
+
+        const appReq: Req = {
+            meta: message.meta,
+            body: message,
+            params: message.params,
+            method: method,
+            path: req.path,
+            url: req.url,
+            headers: req.headers,
+            query: req.query,
+            on: req.on,
+        }
+        const appRes: Res = res as Res
+
         if (method === "get") {
             try {
-                req.body = {...req.body, topic}
-                await (feature.handler as Query)(req, res)
-            } catch (err) {
+                // await feature.handler({...req, headers: req.headers, body: {...req.body, topic} }, res)
+                // req.body = {...req.body, topic}
+                await (feature.handler as Query)(appReq, appRes)
+            } catch (err: any) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
                 throw {status: 500, message: "Failed to GET feature", ...err}
             }
         } else if (method !== "get" && feature.bypass) {
             try {
-                await (feature.handler as Command)(topic, prepareMessage(req), req, res)
-            } catch (err) {
+                await (feature.handler as Command)(topic, message, appReq, appRes)
+            } catch (err: any) {
                 if (process.env.DEBUG_MODE)
-                    console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
+                    console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
                 throw {status: 500, message: "Failed to POST feature", ...err}
             }
-        } else
-            await commandAction(req, res, topic)
-    } catch (err) {
+        } else {
+            try {
+                await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP })
+            } catch {
+                throw {status: 500, message: "Failed to queue the transaction"}
+            }
+            appRes.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } })
+        }
+    } catch (err: any) {
         if (process.env.DEBUG_MODE)
-            console.log('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
+            console.error('Error: Feature: ', req.body?.feature ?? (featureless ? `Featureless Api:${featureless.api}` : undefined), err)
         try { // Sometimes error occurs when disconnecting stream from front end
             res.status(err.status ?? 500).json(err)
         } catch (_) { }
     }
 }
 
-let ExpressApp
-let PROJECT_ROOT_URL
-export async function registerFeatures(newFeatures) {
+let ExpressApp: ExpressApplication
+let PROJECT_ROOT_URL: URL
+export async function registerFeatures(newFeatures: Record<string, FeatureEntry>) {
     // Registering handlers
     await registerHandler(newFeatures)
 
@@ -207,15 +339,15 @@ export async function registerFeatures(newFeatures) {
     // Registering apis
     registerApi()
 }
-async function registerHandler(features) {
+async function registerHandler(features: Record<string, FeatureEntry>) {
     for (let name in features) {
-        let feature = features[name]
+        let feature = features[name]!
         apis[feature.api] = apis[feature.api] ?? {}
-        apis[feature.api][name] = feature
+        apis[feature.api]![name] = feature
         const featureName = name
 
         const nameParts = name.split('.')
-        let handler = nameParts.pop()
+        let handler = nameParts.pop()!
         const featurePath = `src/${nameParts.join('/')}`
 
         if (featurePath === 'src/transactions/query')
@@ -229,8 +361,10 @@ async function registerHandler(features) {
 }
 function registerApi() {
     for (let api in apis) {
-        let method = getFeatureMethod(api)
+        let method: HttpMethod = getFeatureMethod(api)
         let url = api.split(' ')[1]
+        if (!url)
+            throw new Error(`No url path found in api ${api} during Features.registerApi()`)
         if (api === 'GET /transactions/:id')
             continue
 
@@ -238,22 +372,22 @@ function registerApi() {
     }
 }
 
-export const start = async (app, url, file) => {
+export const start = async (app: ExpressApplication, url: URL, file: string) => {
     const globalMeta = Dip.globalMeta()
     Dip.excludeCollectionConfig("users.txns")
     Dip.excludeCollectionConfig("Features.txns")
     ExpressApp = app
     PROJECT_ROOT_URL = url
-    features = await Utils.fileToJson(url, file)
+    features = await Utils.fileToJson(url, file) as Record<string, FeatureEntry>
     await announce()
 
-    app.get('/features', async (req, res) => {
-        let exp_features = {}
+    app.get('/features', async (req: ExpressRequest, res: ExpressResponse) => {
+        let exp_features: Record<string, RemoteFeatureEntry> = {}
         for (let [key, {api}] of Object.entries(features))
-            exp_features[key] = {api, service: `${SERVICE_ADDRESS}`}
+            exp_features[key] = {api, service: `${SERVICE_ADDRESS}`, subscribe: '', topic: ''}
         if (process.env.LOAD_LOCAL_FEATURES) {
             for (let key in SLYP_FEATURES_LIST)
-                SLYP_FEATURES_LIST[key].headers = {JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN}
+                SLYP_FEATURES_LIST[key]!.headers = {JWT: SERVICE_ACCESS_TOKEN, service: true, NDCURVE_DEVELOPER_LICENSE_ACCESS_TOKEN}  // TODO: CRITICAL: Security issue
         }
         res.json({ data: { ...SLYP_FEATURES_LIST, ...exp_features } })
     })
@@ -262,23 +396,24 @@ export const start = async (app, url, file) => {
     await registerFeatures(features)
 
     // Get unique kafka topics list from features
-    let kafkaTopics = []
-    for (let name in features) {
-        let feature = features[name]
-        if (feature.api.split(' ')[0].toLowerCase() === "get" )
+    let kafkaTopics: string[] = []
+    for (const name in features) {
+        let feature = features[name]!
+        if (feature.api.split(' ')[0]?.toLowerCase() === "get" )
             continue
         kafkaTopics = [... new Set(kafkaTopics.concat([feature.topic]))]
     }
 
     // Subscribe to each topic
     for (let topic of kafkaTopics) {
-        Kafka.receive(`Features.${topic}`, async (topic, message) => {
+        const groupId = ''
+        Kafka.receive<FeatureMessage>(`Features.${topic}`, groupId, async (topic: string, message: FeatureMessage) => {
             if (Utils.isEmpty(message.meta?.company))
                 message.meta = { ...message.meta, company: "GLOBAL", outlet: "GLOBAL" }
             if (!message?.feature)
                 return
 
-            const timer = ms => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
+            const timer = (ms: number) => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
             topic = topic.replace(/^.*Features./,'')
             while (true) {
                 try {
@@ -286,7 +421,12 @@ export const start = async (app, url, file) => {
                     // // TODO: use $useChunks: [] once support is added in dip insert
                     // await Dip.insert(globalMeta, `users.txns`, { _id: `${message.user}_${iso_date}_${message.txn}`, user: message.user, feature: message.feature, date: message.date, created: message.date, updated: message.date, status: "Queued" }, {idempotent: true}) // Can always update the status later
                     // TODO: Avoid duplicate processing
-                    await (features[message.feature].handler as Command)(topic, message)
+                    const feature = features[message.feature]
+                    if (!feature) {
+                        console.warn(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`)
+                        throw new Error(`Feature ${message.feature} missing, in executing handler during Kafka.receive(topic: ${topic})`)
+                    }
+                    await (feature.handler as Command)(topic, message)
                     await Dip.insert(globalMeta, "Features.txns", {_id: message.txn, status: "Processed"}, {idempotent: true})
                     break
                 } catch(err) {
@@ -302,9 +442,9 @@ export const start = async (app, url, file) => {
     await subscribe()
 
     // Transaction status api
-    app.get('/transactions/:id', async (req, res) => { // Front end calls this as a regular feature call. so `req.meta` exists
+    app.get('/transactions/:id', async (req: ExpressRequest, res: ExpressResponse) => { // Front end calls this as a regular feature call. so `req.meta` exists
         try {
-            let items = await Dip.query(globalMeta, "Features.txns", {_id: req.params.id})
+            let items = await Dip.query(globalMeta, "Features.txns", {_id: req.params!.id})
             if (items.length)
                 res.json({ data: { ...items[0], txn: items._id } })
             else
@@ -325,70 +465,61 @@ export const start = async (app, url, file) => {
     //     }
     // })
 
-    try { await Messaging.start(app) } catch { console.log('Messaging failed to start. Maybe missing redis') }
+    try { await Messaging.start((app as unknown) as Messaging.ExpressApp ) } catch { console.log('Messaging failed to start. Maybe missing redis') }
 }
 
-function prepareMessage(req) {
-    let txn = req.body.txn ?? Utils.uid()
-    let _id = req.body.data?._id ?? txn
-    let {data, feature, app, user, outlet, company, client, version} = req.body
-    let meta = req.meta
-    let params = req.params
-    return { data, params, meta, feature, app, user, outlet, company, client, version, txn, id: _id, date: new Date().getTime() }
-}
-const commandAction = async (req, res, topic) => {
-    let message = prepareMessage(req)
-    try {
-        await Kafka.send(`Features.${topic}`, message, message.user, { compression: Kafka.CompressionTypes.GZIP })
-    } catch {
-        throw {status: 500, message: "Failed to queue the transaction"}
-    }
-    res.json({ data: { txn: message.txn, success: true, status: "Queued", featureQueued: true } })
-}
 
 
 
-let subscriptions = {
+let subscriptions: Record<string, string> = {
     // "coins.query.hello": "invoices.command.add",     // Consumer : Feature of Topic
 }
-let subscribed_consumers = {
-}
+let subscribed_consumers: Record<string, Kafka.KafkaConsumer> = {}
 async function subscribe() {
     let Features = {get,send}
 
-    for (let [key, {subscribe}] of Object.entries(features))
+    for (let [key, {subscribe}] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe) && subscriptions[key] !== subscribe && subscribed_consumers[key]) {
             try { await subscribed_consumers[key].disconnect() } catch (_) {}
             delete subscribed_consumers[key]
         }
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe) && subscriptions[key] !==  SLYP_FEATURES_LIST[key].subscribe && subscribed_consumers[key]) {
+    }
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key]
+        if (!Utils.isEmpty(feature?.subscribe) && subscriptions[key] !==  feature?.subscribe && subscribed_consumers[key]) {
             try { await subscribed_consumers[key].disconnect() } catch (_) {}
             delete subscribed_consumers[key]
         }
+    }
 
 
-    for (let key in SLYP_FEATURES_LIST)
-        if (!Utils.isEmpty(SLYP_FEATURES_LIST[key].subscribe)) {
-            subscriptions[key] = SLYP_FEATURES_LIST[key].subscribe
+    for (let key in SLYP_FEATURES_LIST) {
+        const feature = SLYP_FEATURES_LIST[key]
+        if (!Utils.isEmpty(feature?.subscribe)) {
+            subscriptions[key] = feature!.subscribe
         }
-    for (let [key, {subscribe}] of Object.entries(features))
+    }
+    for (let [key, {subscribe}] of Object.entries(features)) {
         if (!Utils.isEmpty(subscribe)) {
             subscriptions[key] = subscribe
         }
+    }
 
 
     for (let consumer in subscriptions) {
-        let publisher = subscriptions[consumer]
-        let {topic} = getFeature(publisher)
+        let publisher = subscriptions[consumer]!
+
+        const publisherFeature = getFeature(publisher)
 
-        if (subscribed_consumers[consumer] || !getFeature(publisher))
+        if (subscribed_consumers[consumer] || !publisherFeature)
             continue
 
-        let kafka_consumer = Kafka.receive(`Features.${topic}`, consumer, async (topic, message) => {
+        let {topic} = publisherFeature
+
+        let kafka_consumer_promise = Kafka.receive<FeatureMessage>(`Features.${topic}`, consumer, async (topic: string, message: FeatureMessage) => {
             topic = topic.replace(/^.*Features./,'')
 
-            const timer = ms => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
+            const timer = (ms: number): Promise<void> => new Promise(res => setTimeout(res, ms)) // A promise that resolves after "ms" Milliseconds
 
             while (true) {
                 try {
@@ -400,6 +531,12 @@ async function subscribe() {
                 await timer(10000);
             }
         })
-        subscribed_consumers[consumer] = kafka_consumer
+
+        kafka_consumer_promise
+        .then(kafka_consumer => {
+            subscribed_consumers[consumer] = kafka_consumer
+        }).catch(err => {
+            throw new Error("Error: Remote service to service kafka Feature subscription.")
+        })
     }
 }

package/libs/kafka.ts

@@ -4,12 +4,74 @@ import { Kafka, logLevel as KafkaLogLevel, CompressionTypes as KafkaCompressionT
 export const logLevel = KafkaLogLevel
 export const CompressionTypes = KafkaCompressionTypes
 
-let kafka, producer, admin
 
-let appName
+export type KafkaCallback<T = unknown> = (topic: string, message: T, partition: number) => unknown | Promise<unknown>
+
+export type KafkaConsumer = {
+    connect: () => Promise<void>
+    disconnect: () => Promise<void>
+    subscribe: (arg: {topic: string; fromBeginning: boolean}) => Promise<void>
+    run: (arg: {
+        autoCommit: boolean
+        eachMessage: (arg: {
+            topic: string
+            partition: number
+            message: {
+                value: string | Buffer | null
+                offset: string
+            }
+        }) => Promise<void>
+    }) => Promise<void>
+    commitOffsets: (offsets: {topic: string; partition: number; offset: string}[]) => Promise<void>
+}
+
+type KafkaProducer = {
+    connect: () => Promise<void>
+    send: (arg: unknown) => Promise<unknown>
+}
+
+type KafkaAdmin = {
+    createTopics: (arg: {
+        topics: {
+            topic: string
+            numPartitions: number
+            replicationFactor: number
+        }[]
+    }) => Promise<unknown>
+}
+
+type KafkaClient = {
+    consumer: (arg: {groupId: string}) => KafkaConsumer
+    producer: (arg: {allowAutoTopicCreation: boolean}) => KafkaProducer
+    admin: () => KafkaAdmin
+}
+
+type KafkaStartArg = {
+    clientId?: string
+    brokers?: string[]
+    sasl?: false | {
+        mechanism: string
+        username?: string
+        password?: string
+    }
+    logLevel?: unknown
+}
+
+
+let kafka: KafkaClient
+let producer: KafkaProducer
+let admin: KafkaAdmin
+
+let appName: string
 let topicPrefix = ''
 
-export const start = (arg) => {
+declare global {
+    var app: {
+        name?: string
+    } | undefined
+}
+
+export const start = (arg?: KafkaStartArg) => {
     arg = !arg && !process.env.APP_DEPLOYMENT_NAME ? { clientId: (global.app?.name ?? 'app') + '-dev', brokers: ['127.0.0.1:9092'], sasl: false, logLevel: logLevel.ERROR } : arg
     appName = arg?.clientId ?? (process.env.APP_DEPLOYMENT_NAME ?? (global.app?.name ?? 'app'))
     topicPrefix = `${appName}.`
@@ -28,7 +90,7 @@ export const start = (arg) => {
 }
 
 
-export const createTopic = async (topic, partition, replicas) => {
+export const createTopic = async (topic: string, partition?: number, replicas?: number) => {
     topic = topicPrefix + topic
     return await admin.createTopics({ topics: [{topic, numPartitions: partition ?? 1, replicationFactor: replicas ?? 1}] })
 }
@@ -39,11 +101,10 @@ export const createTopic = async (topic, partition, replicas) => {
 
 let consumers = []
 
-const start_consumer = async function (topic, groupId, callback?: Function) {
-    callback = typeof groupId === "string" ? callback : groupId
-    groupId = typeof groupId === "string" ? `${appName}.${groupId}` : `${appName}.${topic}`
+const start_consumer = async function<T>(topic: string, groupId: string, callback: KafkaCallback<T>): Promise<KafkaConsumer> {
+    groupId = groupId ? `${appName}.${groupId}` : `${appName}.${topic}`
 
-    const consumer = kafka.consumer({ groupId: groupId })
+    const consumer: KafkaConsumer = kafka.consumer({ groupId: groupId })
     consumers.push(consumer)
 
     await consumer.connect()
@@ -55,9 +116,9 @@ const start_consumer = async function (topic, groupId, callback?: Function) {
             // callback(topic, message.value.toString(), partition) // toString() returns array so won't parse if json.
             let success
             try {
-                success = await callback(topic, JSON.parse(message.value), partition)
+                success = await callback(topic, JSON.parse(message.value as string), partition)
             } catch (ex) {
-                success = await callback(topic, message.value, partition)
+                success = await callback(topic, message.value as T, partition) // NOTE: Beware: Ensure Kafka.receive<T> can handle raw message type that is not json parseable
             }
 
             await consumer.commitOffsets([{ topic, partition, offset: (Number(message.offset) + 1).toString() }]);
@@ -71,7 +132,7 @@ const start_consumer = async function (topic, groupId, callback?: Function) {
 // Producer
 // ===========
 let producerInvoked = false
-const start_producer = async function (topic, message, key, options) {
+const start_producer = async function (topic: string, message: object, key: string, options?: object) {
 
     let isJson = Object.prototype.toString.call(message) === '[object Object]' || Object.prototype.toString.call(message) === '[object Array]'
 
@@ -97,13 +158,13 @@ const start_producer = async function (topic, message, key, options) {
 // start_consumer('quickstart-events')
 // start_producer('quickstart-events', 'Hello KafkaJS user! Little')
 
-export const receive = async function(topic, groupId, callback?: Function) {
+export const receive = async function<T>(topic: string, groupId: string, callback: KafkaCallback<T>): Promise<KafkaConsumer> {
     topic = topicPrefix + topic
-    return (await start_consumer(topic, groupId, callback))
+    return await start_consumer(topic, groupId, callback)
 }
 
 
-export const send = async function(topic, message, key, options) {
+export const send = async function(topic: string, message: object, key: string, options?: object) {
     topic = topicPrefix + topic
     await start_producer(topic, message, key, options)
 }