@dataloop-ai/components 0.20.127 → 0.20.128

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dataloop-ai/components",
-  "version": "0.20.127",
+  "version": "0.20.128",
   "exports": {
     ".": "./index.ts",
     "./models": "./models.ts",
@@ -37,6 +37,7 @@
     "sass": "^1.51.0",
     "sass-loader": "^12.6.0",
     "sortablejs": "^1.15.0",
+    "tokenizr": "^1.7.0",
     "uuid": "^8.3.2",
     "v-wave": "^1.5.0",
     "vanilla-jsoneditor": "^0.10.2",

package/src/components/compound/DlSearches/DlSmartSearch/components/DlSmartSearchInput.vue

@@ -181,6 +181,7 @@ import {
 } from '../../../../../hooks/use-suggestions'
 import { parseSmartQuery, stringifySmartQuery } from '../../../../../utils'
 import { StateManager, stateManager } from '../../../../../StateManager'
+import { TokenType, tokenize } from '../../../../../utils/splitByQuotes'
 
 export default defineComponent({
     components: {
@@ -428,68 +429,63 @@ export default defineComponent({
             const value = '' + suggestion
             let stringValue = ''
             let caretPosition = 0
-            if (searchQuery.value.length) {
-                let queryLeftSide = searchQuery.value.substring(
-                    0,
-                    caretAt.value
-                )
-                let queryRightSide = searchQuery.value.substring(caretAt.value)
 
+            const search = searchQuery.value ?? ''
+            const tokens = tokenize(search)
+            let leftTokenIndex = tokens.length
+            while(leftTokenIndex-- > 0) {
+                if (tokens[leftTokenIndex].pos < caretAt.value) {
+                    break
+                }
+            }
+
+            if (leftTokenIndex < 0) {
+                stringValue = value + ' ' + search.replace(/^\S*\s*/, '')
+                caretPosition = value.length + 1
+            } else {
+                const token = tokens[leftTokenIndex]
+                const tokenLeftText = token.text.substring(0, caretAt.value - token.pos)
+                const tokenRightText = token.text.substring(caretAt.value - token.pos)
+
+                if (token.type === TokenType.WHITESPACE) {
+                    // caret after space
+                    token.text = ' ' + value + ' '
+                    caretPosition = token.pos + 1 + value.length + 1
+                } else
                 if (['AND', 'OR'].includes(value)) {
                     // do not replace text if the value is AND or OR
-                    const leftover = queryLeftSide.match(/\S+$/)?.[0] || ''
-                    queryLeftSide =
-                        queryLeftSide.replace(/\S+$/, '').trimEnd() + ' '
-                    queryRightSide = leftover + queryRightSide
+                    token.text = value + ' ' + token.text
+                    caretPosition = token.pos + value.length + 1
                 } else if (value.startsWith('.')) {
                     // dot notation case
-                    const words = queryLeftSide.trimEnd().split('.')
+                    const words = tokenLeftText.split('.')
                     const lastWord = words.pop()
                     if (!value.startsWith('.' + lastWord)) {
                         words.push(lastWord)
                     }
-                    queryLeftSide = words.join('.')
-                } else if (
-                    queryLeftSide.endsWith(' ') &&
-                    (queryLeftSide.match(/'/g)?.length ?? 0) % 2 === 0
-                ) {
-                    // caret after space: only replace multiple spaces on the left
-                    queryLeftSide = queryLeftSide.trimEnd() + ' '
-                } else if (/\.\S+$/.test(queryLeftSide)) {
-                    // if there are dots in left side expression, suggestions have an operator
+                    const text = words.join('.')
+                    token.text = text + value + ' ' + tokenRightText
+                    caretPosition = token.pos + text.length + value.length + 1
+                } else if (/\.\S+$/.test(tokenLeftText)) {
+                    // if there are dots in left side expression...
                     // looks like a bug in findSuggestions TODO find it - for now work around it here
-                    const leftover = queryRightSide.match(/^\S+/)?.[0] || ''
-                    queryLeftSide += leftover + ' '
-                    queryRightSide = queryRightSide
-                        .substring(leftover.length)
-                        .trimStart()
-                } else if (queryRightSide.startsWith(' ')) {
-                    // this| situation: replace whatever is there on the left side with the value
-                    queryLeftSide = queryLeftSide.replace(/\S+$/, '')
-                    queryRightSide = queryRightSide.trimStart()
+                    const leftover = tokenRightText.match(/^\S+/)?.[0] || ''
+                    token.text = tokenLeftText + leftover + ' ' + value + ' ' +
+                        tokenRightText.substring(leftover.length).trimStart()
+                    caretPosition = token.pos + tokenLeftText.length +
+                        leftover.length + 1 + value.length + 1
                 } else {
+                    // this| situation: replace whatever is there on the left side with the value
                     // this|situation: replace whatever is there on both sides with the value
-                    if (
-                        /^[^']+((?<!\\)'([^']|(?<=\\)')*(?<!\\)'[^']+)*(?<!\\)'([^']+\\')*[^']*((?<!\\)')?$/.test(
-                            queryLeftSide
-                        )
-                    ) {
-                        queryLeftSide = queryLeftSide.replace(
-                            /(?<!\\)'([^']+\\')*[^']*((?<!\\)')?$/,
-                            ''
-                        )
-                    } else {
-                        queryLeftSide = queryLeftSide.replace(/[^'\s]+$/, '')
+                    const newValue = token.type === TokenType.COMMA ? ', ' + value : value
+                    token.text = newValue
+                    caretPosition = token.pos + newValue.length
+                    if (tokens[leftTokenIndex + 1]?.type !== TokenType.WHITESPACE) {
+                        token.text += ' '
+                        caretPosition += 1
                     }
-                    queryRightSide =
-                        removeLeadingExpression(queryRightSide).trimStart()
                 }
-
-                stringValue = queryLeftSide + value + ' ' + queryRightSide
-                caretPosition = stringValue.length - queryRightSide.length
-            } else {
-                stringValue = value + ' '
-                caretPosition = stringValue.length
+                stringValue = tokens.map(token => token.text).join('')
             }
 
             setInputValue(stringValue)

package/src/components/compound/DlSearches/DlSmartSearch/utils/highlightSyntax.ts

@@ -1,3 +1,4 @@
+import { tokenize } from '../../../../../utils/splitByQuotes'
 import { SyntaxColorSchema } from '../types'
 
 const SPAN_STYLES = `overflow: hidden;
@@ -91,7 +92,7 @@ function restoreSelection(
 }
 
 function renderText(text: string, colorSchema: SyntaxColorSchema) {
-    const words = text?.split(/(\s+)/)
+    const words = tokenize(text ?? '').map(token => token.text)
     const output = words?.map((word) => {
         if (colorSchema) {
             if (colorSchema.keywords.values.includes(word)) {

package/src/hooks/use-suggestions.ts

@@ -1,5 +1,6 @@
 import { Ref, ref } from 'vue-demi'
-import { splitByQuotes } from '../utils/splitByQuotes'
+import { splitByQuotes, tokenize, TokenType } from '../utils/splitByQuotes'
+import { Token } from 'tokenizr'
 import { flatten } from 'flat'
 import { isObject } from 'lodash'
 
@@ -192,12 +193,56 @@ export const useSuggestions = (
     }
 
     const findSuggestions = (input: string) => {
-        input = input.replace(/\s+/g, ' ').trimStart()
-        localSuggestions = sortedSuggestions
+        const tokens = tokenize(input)
+
+        let fieldToken: Token | null = null
+        let operatorToken: Token | null = null
+        let valueToken: Token | null = null
+        let keywordToken: Token | null = null
+
+        let i = tokens.length -1
+        let whitespace = false
+        while (i > -1) {
+            const token = tokens[i]
+            switch (token.type) {
+                case TokenType.WHITESPACE:
+                    whitespace = true
+                    break
+                case TokenType.LOGICAL:
+                    keywordToken = token
+                    break
+                case TokenType.BOOLEAN:
+                case TokenType.COMMA:
+                case TokenType.DATETIME:
+                case TokenType.NUMBER:
+                case TokenType.STRING:
+                case TokenType.PARTIAL_VALUE:
+                    if (!valueToken) {
+                        valueToken = token
+                    }
+                    break
+                case TokenType.OPERATOR:
+                    // quirk: mapWordsToExpression would only set operator if followed by a whitespace
+                    if (whitespace || valueToken) {
+                        operatorToken = token
+                    }
+                    break
+                case TokenType.FIELD:
+                    fieldToken = token
+                    i = 0
+                    break
+            }
+            i--
+        }
 
-        const words = splitByQuotes(input, space)
-        const mergedWords = mergeWords(words)
-        const expressions = mapWordsToExpressions(mergedWords)
+        const expressions: Expression[] = [{
+            field: fieldToken?.text,
+            operator: operatorToken?.text,
+            value: valueToken?.type === TokenType.COMMA ? '' : valueToken?.text,
+            keyword: keywordToken?.text
+        }]
+                
+        localSuggestions = sortedSuggestions
 
         for (const { field, operator, value, keyword } of expressions) {
             let matchedField: Suggestion | null = null
@@ -228,7 +273,9 @@ export const useSuggestions = (
 
             if (
                 !matchedField ||
-                (!isNextCharSpace(input, matchedField) &&
+                (
+                    !operator &&
+                    insensitive(input).endsWith(insensitive(matchedField)) &&
                     fieldSeparated.length === 1)
             ) {
                 continue
@@ -328,7 +375,10 @@ export const useSuggestions = (
                 continue
             }
 
-            if (!matchedOperator || !isNextCharSpace(input, matchedOperator)) {
+            if (!matchedOperator || (
+                !value &&
+                !isNextCharSpace(input, matchedOperator)
+            )) {
                 continue
             }
 

package/src/utils/splitByQuotes.ts

@@ -1,41 +1,112 @@
-export function splitByQuotes(input: string, split: string) {
-    const pattern = '([\\s\\S]*?)(e)?(?:(o)|(c)|(t)|(sp)|$)'
-        .replace('sp', split)
-        .replace('o', '[\\(\\{\\[]')
-        .replace('c', '[\\)\\}\\]]')
-        .replace('t', '(?<!\\\\)[\'"]')
-        .replace('e', '[\\\\]')
-    const r = new RegExp(pattern, 'gi')
-    const stack: string[] = []
-    let buffer: string[] = []
-    const results: string[] = []
-    input.replace(r, ($0, $1, $e, $o, $c, $t, $s, i): any => {
-        if ($e) {
-            buffer.push($1, $s || $o || $c || $t)
-            return
-        } else if ($o) {
-            if (!stack.includes("'") && !stack.includes('"')) {
-                stack.push($o)
-            }
-        } else if ($c) {
-            if (!stack.includes("'") && !stack.includes('"')) {
-                stack.pop()
-            }
-        } else if ($t) {
-            const otherQuote = $t === '"' ? "'" : '"'
-            if (!stack.includes(otherQuote)) {
-                if (stack[stack.length - 1] !== $t) stack.push($t)
-                else stack.pop()
-            }
+import Tokenizr from "tokenizr"
+
+export enum TokenType {
+    NUMBER = 'number',
+    DATETIME = 'datetime',
+    OPERATOR = 'operator',
+    LOGICAL = 'logical',
+    BOOLEAN = 'boolean',
+    FIELD = 'field',
+    COMMA = 'comma',
+    STRING = 'string',
+    PARTIAL_VALUE = 'partial-string',
+    WHITESPACE = 'whitespace'
+}
+
+enum Tags {
+    HAD_FIELD = 'had-field',
+    HAD_VALUE = 'had-value'
+}
+
+let tokenizer = new Tokenizr()
+
+tokenizer.rule(/[+-]?[0-9\.]+/, (ctx, match) => {
+    ctx.accept(TokenType.NUMBER, parseFloat(match[0])).tag(Tags.HAD_VALUE)
+})
+
+tokenizer.rule(/\((\d{2}\/\d{2}\/\d{4}[\)']?\s?|\s?DD\/MM\/YYYY)\s?(\d{2}:\d{2}:\d{2}|\s?HH:mm:ss)?\)/, (ctx, match) => {
+    ctx.accept(TokenType.DATETIME, parseFloat(match[0])).tag(Tags.HAD_VALUE)
+})
+
+;[
+    /<=?/, />=?/, /!=?/, /=/,
+    /in?(?![^\s'"])/i,
+    /n(o(t(-(in?)?)?)?)?(?![^\s'"])/i,
+    /e(x(i(s(ts?)?)?)?)?(?!\S)/i,
+    /d(o(e(s(n(t(-(e(x(i(st?)?)?)?)?)?)?)?)?)?)?(?!\S)/i
+].forEach(re => tokenizer.rule(re, (ctx, match) => {
+    if (!ctx.tagged(Tags.HAD_FIELD) && /^[a-z]/i.test(match[0])) {
+        ctx.accept(TokenType.FIELD).tag(Tags.HAD_FIELD)
+    } else {
+        ctx.accept(TokenType.OPERATOR, match[0].toUpperCase())
+    }
+}))
+
+tokenizer.rule(/[a-z][a-z\.\d\-_]*/i, (ctx, match) => {
+    const upper = match[0].toUpperCase()
+    if (ctx.tagged(Tags.HAD_VALUE)) {
+        // we just had a value - it would be followed by AND or OR
+        ctx.untag(Tags.HAD_FIELD).untag(Tags.HAD_VALUE).accept(TokenType.LOGICAL, upper)
+    } else if (ctx.tagged(Tags.HAD_FIELD)) {
+        // we had a field but no value yet - this must be either a boolean or an unquoted string
+        if (['TRUE', 'FALSE'].includes(upper)) {
+            ctx.accept(TokenType.BOOLEAN, upper === 'TRUE').tag(Tags.HAD_VALUE)
         } else {
-            if ($s ? !stack.length : !$1) {
-                buffer.push($1)
-                results.push(buffer.join(''))
-                buffer = []
-                return
-            }
+            ctx.accept(TokenType.PARTIAL_VALUE).tag(Tags.HAD_VALUE)
         }
-        buffer.push($0)
-    })
-    return results
+    } else {
+        ctx.accept(TokenType.FIELD).tag(Tags.HAD_FIELD)
+    }
+})
+
+tokenizer.rule(/,/, (ctx) => {
+    // untag HAD_VALUE because a comma cannot be followed by AND or OR - but by another value
+    ctx.untag(Tags.HAD_VALUE).accept(TokenType.COMMA)
+})
+
+tokenizer.rule(/(?<!\\)"(.*?)(?<!\\)"/, (ctx, match) => {
+    ctx.accept(TokenType.STRING, match[1].replace(/\\"/g, '"')).tag(Tags.HAD_VALUE)
+})
+
+tokenizer.rule(/(?<!\\)'(.*?)(?<!\\)'/, (ctx, match) => {
+    ctx.accept(TokenType.STRING, match[1].replace(/\\'/g, "'")).tag(Tags.HAD_VALUE)
+})
+
+tokenizer.rule(/(?<!\\)['"](.*)/, (ctx, match) => {
+    // partial string
+    ctx.accept(TokenType.PARTIAL_VALUE, match[1].replace(/\\'/g, "'")).tag(Tags.HAD_VALUE)
+})
+
+tokenizer.rule(/\s+/, (ctx) => {
+    ctx.accept(TokenType.WHITESPACE)
+})
+
+tokenizer.rule(/.+/, (ctx, match) => {
+    // unrecognized token
+    ctx.accept(TokenType.WHITESPACE, match[0].replaceAll(/./g, ' '))
+})
+
+export function tokenize(input: string) {
+    tokenizer.reset()
+    tokenizer.input(input)
+    return tokenizer.tokens()
+}
+
+export function splitByQuotes(input: string, ignore?: string) {
+    const parts = tokenize(input)
+        .filter(token => token.type !== 'whitespace')
+        .map(token => token.text)
+        .map((text, index, array) => {
+            if(array[index + 1] === ',') {
+                return text + ','
+            } else {
+                return text === ',' ? '' : text
+            }
+        })
+        .filter(text => text !== '')
+
+    if (/\s$/.test(input)) {
+        parts.push('')
+    }
+    return parts
 }