@cap-js/sqlite 0.1.0

package/lib/db/sql/deep.js

@@ -0,0 +1,233 @@
+const cds = require('@sap/cds')
+const { getDBTable } = require('@sap/cds/libx/_runtime/common/utils/resolveView')
+const { compareJson } = require('@sap/cds/libx/_runtime/cds-services/services/utils/compareJson')
+const { target_name4 } = require('./utils')
+
+const handledDeep = Symbol('handledDeep')
+
+async function onDeep (req, next) {
+  const { query } = req
+  // REVISIT: req.target does not match the query.INSERT target for path insert
+  // cds.inferred(query)
+  // const target = query.sources[Object.keys(query.sources)[0]]
+  if (!this.model?.definitions[target_name4(req.query)]) {
+    return next()
+  }
+  const {target} = this.infer(query)
+  if (!hasDeep(query, target)) return next()
+  const beforeData = query.INSERT ? [] : await this.run(getExpandForDeep(query, target, true))
+  const queries = getDeepQueries(query, beforeData, target)
+  const res = await Promise.all(queries.map (q => this.run(q)))
+  return res[0] ?? 0 // TODO what todo with multiple result responses?
+}
+
+
+const hasDeep = (query, target) => {
+  if (handledDeep in query) return
+  if (query.DELETE) {
+    for (let c in target?.compositions) return true
+    return false
+  }
+  const data = query.INSERT?.entries || (query.UPDATE?.data && [query.UPDATE.data]) || (query.UPDATE?.with && [query.UPDATE.with])
+  if (data) for (const c in target.compositions) {
+    for (const row of data) if (row[c] !== undefined) return true
+  }
+}
+
+// unofficial config!
+const DEEP_DELETE_MAX_RECURSION_DEPTH =
+  (cds.env.features.recursion_depth && Number(cds.env.features.recursion_depth)) || 4 // we use 4 here as our test data has a max depth of 3
+
+// IMPORTANT: Skip only if @cds.persistence.skip is `true` → e.g. this skips skipping targets marked with @cds.persistence.skip: 'if-unused'
+const _hasPersistenceSkip = target => target?.["@cds.persistence.skip"] === true
+
+const getColumnsFromDataOrKeys = (data, target) => {
+  if (Array.isArray(data)) {
+    // loop and get all columns from current level
+    const columns = new Set()
+    data.forEach(row =>
+      Object.keys(row || target.keys)
+        .filter(propName => !target.elements[propName]?.isAssociation)
+        .forEach(entry => {
+          columns.add(entry)
+        })
+    )
+    return Array.from(columns).map(c => ({ ref: [c] }))
+  } else {
+    // get all columns from current level
+    return Object.keys(data || target.keys)
+      .filter(propName => !target.elements[propName]?.isAssociation)
+      .map(c => ({ ref: [c] }))
+  }
+}
+
+const _calculateExpandColumns = (compositions, data, expandColumns = [], elementMap = new Map()) => {
+  for (const compName in compositions) {
+    let compositionData
+    if (data === null) {
+      compositionData = null
+    } else {
+      compositionData = data[compName]
+    }
+    // ignore not provided compositions as nothing happens with them (expect deep delete)
+    if (compositionData === undefined) {
+      // fill columns in case
+      continue
+    }
+
+    const composition = compositions[compName]
+
+    const fqn = composition.parent.name + ':' + composition.name
+    const seen = elementMap.get(fqn)
+    if (seen && seen >= DEEP_DELETE_MAX_RECURSION_DEPTH) {
+      // recursion -> abort
+      return
+    }
+
+    let expandColumn = expandColumns.find(expandColumn => expandColumn.ref[0] === composition.name)
+    if (!expandColumn) {
+      expandColumn = {
+        ref: [composition.name],
+        expand: getColumnsFromDataOrKeys(compositionData, composition._target)
+      }
+      if (expandColumns.length === 0) {
+        expandColumns.push(...getColumnsFromDataOrKeys(data, composition.parent))
+      }
+
+      expandColumns.push(expandColumn)
+    }
+
+    const targetCompositions = composition._target.compositions
+
+    // expand deep
+    if (targetCompositions) {
+      // Make a copy and do not share the same map among brother compositions
+      // as we're only interested in deep recursions, not wide recursions.
+      const newElementMap = new Map(elementMap)
+      newElementMap.set(fqn, (seen && seen + 1) || 1)
+
+      if (composition.is2many) {
+        // expandColumn.expand = getColumnsFromDataOrKeys(compositionData, composition._target)
+        if (compositionData === null || compositionData.length === 0) {
+          // deep delete, get all subitems until recursion depth
+          _calculateExpandColumns(targetCompositions, compositionData, expandColumn.expand, newElementMap)
+          return
+        }
+
+        for (const row of compositionData) {
+          _calculateExpandColumns(targetCompositions, row, expandColumn.expand, newElementMap)
+        }
+      } else {
+        // to one
+        _calculateExpandColumns(targetCompositions, compositionData, expandColumn.expand, newElementMap)
+      }
+    }
+  }
+}
+
+const getExpandForDeep = (query, target) => {
+  const from = query.DELETE?.from || query.UPDATE?.entity
+  const data = query.UPDATE?.data || null
+  const where = query.DELETE?.where || query.UPDATE?.where
+
+  const cqn = SELECT.from(from)
+  if (where) cqn.SELECT.where = where
+
+  const compositions = target.compositions || {}
+
+  const columns = []
+  _calculateExpandColumns(compositions, data, columns)
+  cqn.columns(columns)
+  return cqn
+}
+
+const getDeepQueries = (query, dbData, target) => {
+  let queryData
+  if (query.INSERT) {
+    queryData = query.INSERT.entries
+  }
+  if (query.DELETE) {
+    queryData = []
+  }
+  if (query.UPDATE) {
+    queryData = [query.UPDATE.data]
+  }
+
+  let diff = compareJson(queryData, dbData, target)
+  if (!Array.isArray(diff)) {
+    diff = [diff]
+  }
+
+  return _getDeepQueries(diff, target)
+}
+
+const _getDeepQueries = (diff, target) => {
+  const queries = []
+
+  for (const diffEntry of diff) {
+    if (diffEntry === undefined) continue
+    const subQueries = []
+
+    for (const prop in diffEntry) {
+      // handle deep operations
+
+      const propData = diffEntry[prop]
+
+      if (target.elements[prop] && _hasPersistenceSkip(target.elements[prop]._target)) {
+        delete diffEntry[prop]
+      } else if (propData && propData._op) {
+        // to-one can also contain deep, so call recursively
+        subQueries.push(..._getDeepQueries([propData], target.elements[prop]._target))
+        delete diffEntry[prop]
+      } else if (Array.isArray(propData)) {
+        // REVISIT can be arrayed too
+        propData.forEach(subEntry => {
+          if (subEntry._op) {
+            subQueries.push(..._getDeepQueries([subEntry], target.elements[prop]._target))
+          }
+        })
+        delete diffEntry[prop]
+      }
+    }
+
+    // handle current entity level
+    const op = diffEntry._op
+    delete diffEntry._op
+
+    if (diffEntry._old != null) {
+      delete diffEntry._old
+    }
+
+    // first calculate subqueriesand rm their properties, then build root query
+    if (op === 'create') {
+      queries.push(INSERT.into(getDBTable(target)).entries(diffEntry))
+    } else if (op === 'delete') {
+      queries.push(DELETE.from(getDBTable(target)).where(diffEntry))
+    } else if (op === 'update') {
+      // TODO do we need the where here?
+      const keys = target.keys
+      const cqn = UPDATE(getDBTable(target)).with(diffEntry)
+      for (const key in keys) {
+        if (!keys[key].isAssociation) {
+          cqn.where(key + '=', diffEntry[key])
+        }
+        delete diffEntry[key]
+      }
+      cqn.with(diffEntry)
+      queries.push(cqn)
+    }
+
+    queries.push(...subQueries)
+  }
+
+  queries.forEach(q => {
+    Object.defineProperty(q, handledDeep, { value: true })
+  })
+  return queries
+}
+
+module.exports = {
+  onDeep,
+  getDeepQueries,
+  getExpandForDeep
+}

package/lib/db/sql/func.js

@@ -0,0 +1,146 @@
+const StandardFunctions = {
+
+  // OData: https://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part2-url-conventions.html#sec_CanonicalFunctions
+
+  // String and Collection Functions
+  // length     : (x) => `length(${x})`,
+  search: (ref, arg) => {
+    if (!('val' in arg)) throw `SQLite only supports single value arguments for $search`
+    const refs = ref.list || [ref], {toString} = ref
+    return '(' + refs.map(ref2 => `ifnull(instr(lower(${toString(ref2)}),lower(${arg})),0)`).join(' or ') + ')'
+  },
+  concat     : (...args) => args.join('||'),
+  contains   : (...args) => `ifnull(instr(${args}),0)`,
+  count      : (x)   => `count(${x||'*'})`,
+  indexof    : (x,y) => `instr(${x},${y}) - 1`, // sqlite instr is 1 indexed
+  startswith : (x,y) => `instr(${x},${y}) = 1`, // sqlite instr is 1 indexed
+  endswith   : (x,y) => `instr(${x},${y}) = length(${x}) - length(${y}) +1`,
+  substring  : (x,y,z) => ( z
+    ? `substr( ${x}, case when ${y} < 0 then length(${x}) + ${y} + 1 else ${y} + 1 end, ${z} )`
+    : `substr( ${x}, case when ${y} < 0 then length(${x}) + ${y} + 1 else ${y} + 1 end )`
+  ),
+
+  // String Functions
+  matchesPattern : (x,y) => `${x} regexp ${y})`,
+  tolower        : (x) => `lower(${x})`,
+  toupper        : (x) => `upper(${x})`,
+  // trim           : (x) => `trim(${x})`,
+
+  // Arithmetic Functions
+  ceiling  : (x) => `ceil(${x})`,
+  // floor    : (x) => `floor(${x})`,
+  // round    : (x) => `round(${x})`,
+
+  // Date and Time Functions
+  year     : (x) => `cast( strftime('%Y',${x}) as Integer )`,
+  month    : (x) => `cast( strftime('%m',${x}) as Integer )`,
+  day      : (x) => `cast( strftime('%d',${x}) as Integer )`,
+  hour     : (x) => `cast( strftime('%H',${x}) as Integer )`,
+  minute   : (x) => `cast( strftime('%M',${x}) as Integer )`,
+  second   : (x) => `cast( strftime('%S',${x}) as Integer )`,
+
+  fractionalseconds : (x) => `cast( strftime('%f',${x}) as Integer )`,
+
+  maxdatetime : ()=> '9999-12-31 23:59:59.999',
+  mindatetime : ()=> '0001-01-01 00:00:00.000',
+
+  // odata spec defines the date time offset type as a normal ISO time stamp
+  // Where the timezone can either be 'Z' (for UTC) or [+|-]xx:xx for the time offset
+  // sqlite understands this so by splitting the timezone from the actual date
+  // prefixing it with 1970 it allows sqlite to give back the number of seconds
+  // which can be divided by 60 back to minutes
+  totaloffsetminutes : (x) => `case
+    when substr(${x}, length(${x})) = 'z' then 0
+    else strftime('%s', '1970-01-01T00:00:00' || substr(${x}, length(${x}) - 5)) / 60
+  end`,
+
+  // odata spec defines the value format for totalseconds as a duration like: P12DT23H59M59.999999999999S
+  // P -> duration indicator
+  // D -> days, T -> Time seperator, H -> hours, M -> minutes, S -> fractional seconds
+  // By splitting the DT and calculating the seconds of the time separate from the day
+  // it possible to determine the full amount of seconds by adding them together as fractionals and multiplying
+  // the number of seconds in a day
+  // As sqlite is most accurate with juliandays it is better to do then then using actual second function
+  // while the odata specification states that the seconds has to be fractional which only julianday allows
+  totalseconds : (x) => `(
+    (
+      (
+        cast(substr(${x},2,instr(${x},'DT') - 2) as Integer)
+      ) + (
+        julianday(
+          '-4713-11-25T' ||
+          replace(
+          replace(
+          replace(
+            substr(${x},instr(${x},'DT') + 2),
+            'H',':'
+          ),'M',':'
+          ),'S','Z'
+          )
+        ) - 0.5
+      )
+    ) * 86400
+  )`,
+}
+
+
+const HANAFunctions = {
+  // https://help.sap.com/docs/SAP_HANA_PLATFORM/4fe29514fd584807ac9f2a04f6754767/f12b86a6284c4aeeb449e57eb5dd3ebd.html
+
+  // Time functions
+  nano100_between : (x,y) => `(julianday(${y}) - julianday(${x})) * 864000000000`,
+  seconds_between : (x,y) => `(julianday(${y}) - julianday(${x})) * 86400`,
+  // Calculates the difference in full days using julian day
+  // Using the exact time of the day to determine whether 24 hours have passed or not to add the final day
+  // When just comparing the julianday values with each other there are leap seconds included
+  // Which on the day resolution are included as the individual days therefor ignoring them to match HANA
+  days_between : (x,y) => `(
+    cast ( julianday(${y}) as Integer ) - cast ( julianday(${x}) as Integer )
+  ) + (
+    case
+      when ( julianday(${y}) < julianday(${x}) ) then
+        (cast( strftime('%H%M%S%f', ${y}) as Integer ) < cast( strftime('%H%M%S%f', ${x}) as Integer ))
+      else
+        (cast( strftime('%H%M%S%f', ${y}) as Integer ) > cast( strftime('%H%M%S%f', ${x}) as Integer )) * -1
+    end
+  )`,
+
+  // (y1 - y0) * 12 + (m1 - m0) + (t1 < t0) * -1
+  /* '%d%H%M%S%f' returns as a number like which results in an equal check to:
+  (
+    d1 < d0 ||
+    (d1 = d0 && h1 < h0) ||
+    (d1 = d0 && h1 = h0 && m1 < m0) ||
+    (d1 = d0 && h1 = h0 && m1 = m0 && s1 < s0) ||
+    (d1 = d0 && h1 = h0 && m1 = m0 && s1 = s0 && ms1 < ms0)
+  )
+  Which will remove the current month if the time of the month is below the time of the month of the start date
+  It should not matter that the number of days in the month is different as for a month to have passed
+  the time of the month would have to be higher then the time of the month of the start date
+
+  Also check whether the result will be positive or negative to make sure to not subtract an extra month
+  */
+  months_between : (x,y) => `
+  (
+    (
+      ( cast( strftime('%Y', ${y}) as Integer ) - cast( strftime('%Y', ${x}) as Integer ) ) * 12
+    ) + (
+      cast( strftime('%m', ${y}) as Integer ) - cast( strftime('%m', ${x}) as Integer )
+    ) + (
+      (
+        case
+          when ( cast( strftime('%Y%m', ${y}) as Integer ) < cast( strftime('%Y%m', ${x}) as Integer ) ) then
+            (cast( strftime('%d%H%M%S%f', ${y}) as Integer ) > cast( strftime('%d%H%M%S%f', ${x}) as Integer ))
+          else
+            (cast( strftime('%d%H%M%S%f', ${y}) as Integer ) < cast( strftime('%d%H%M%S%f', ${x}) as Integer )) * -1
+        end
+      )
+    )
+  )`,
+  years_between(x,y) { return `floor(${this.months_between(x,y)} / 12)` },
+}
+
+for (let each in HANAFunctions)
+  HANAFunctions[each.toUpperCase()] = HANAFunctions[each]
+
+module.exports = { ...StandardFunctions, ...HANAFunctions }

package/lib/db/sql/structuralComparisonOps.js

@@ -0,0 +1,16 @@
+/**
+  * For operators of <eqOps>, this is replaced by comparing all leaf elements with null, combined with and.
+  * If there are at least two leaf elements and if there are tokens before or after the recognized pattern, we enclose the resulting condition in parens (...)
+  */
+const eqOps = [['is'], ['='], /* ['=='] */ ]
+/**
+  * For operators of <notEqOps>, do the same but use or instead of and.
+  * This ensures that not struc == <value> is the same as struc != <value>.
+  */
+const notEqOps = [['is', 'not'], ['<>'], ['!=']]
+/**
+  * not supported in comparison w/ struct because of unclear semantics
+  */
+const notSupportedOps = [['>'], ['<'], ['>='], ['<=']]
+
+module.exports = { eqOps, notEqOps, notSupportedOps }

package/lib/db/sql/utils.js

@@ -0,0 +1,22 @@
+const cds = require('@sap/cds/lib')
+
+const target_name4 = q => {
+    const target = (
+      q.SELECT?.from ||
+      q.INSERT?.into ||
+      q.UPSERT?.into ||
+      q.UPDATE?.entity ||
+      q.DELETE?.from ||
+      q.CREATE?.entity ||
+      q.DROP?.entity ||
+      undefined
+    )
+    if (target?.SET?.op === "union") throw new cds.error('”UNION” based queries are not supported');
+    if (!target?.ref) return target
+    const [first] = target.ref
+    return first.id || first
+  }
+
+  module.exports = {
+    target_name4
+  }

package/lib/db/sql/workarounds.js

@@ -0,0 +1,73 @@
+const { resolveView } = require('@sap/cds/libx/_runtime/common/utils/resolveView')
+const cds = require('@sap/cds')
+const propagateForeignKeys = require('@sap/cds/libx/_runtime/common/utils/propagateForeignKeys')
+const { enrichDataWithKeysFromWhere } = require('@sap/cds/libx/_runtime/common/utils/keys')
+
+const generateUUIDandPropagateKeys = (target, data, event) => {
+  if (!data) return
+  const keys = target.keys
+  for (const key in keys) {
+    if (keys[key].type === 'cds.UUID' && !data[key] && event === 'CREATE') {
+      data[key] = cds.utils.uuid()
+    }
+  }
+  const elements = target.elements
+  for (const element in elements) {
+    // if assoc keys are structured, do not ignore them, as they need to be flattened in propagateForeignKeys
+    if (elements[element].key && !(elements[element]._isAssociationStrict && elements[element].is2one && element in data)) {
+      continue
+    }
+
+    if (elements[element].is2one || elements[element].is2many) {
+      let subData = data[element]
+      if (subData) {
+        if (!Array.isArray(subData)) {
+          subData = [subData]
+        }
+        for (const sub of subData) {
+          generateUUIDandPropagateKeys(elements[element]._target, sub, event)
+        }
+      }
+
+      propagateForeignKeys(element, data, elements[element]._foreignKeys, elements[element].isComposition, {
+        deleteAssocs: true
+      })
+    }
+  }
+}
+
+const input = async function (req, next) {
+  // REVISIT dummy handler until we have input processing
+  if (!req.target || !this.model) return next()
+
+  if (req.event === "UPDATE") {
+    // REVISIT for deep update we need to inject the keys first
+    enrichDataWithKeysFromWhere(req.data, req, this)
+  }
+
+  // REVISIT no input processing for INPUT with rows/values
+  if (req.event !== 'DELETE' && !(req.query.INSERT?.rows || req.query.INSERT?.values)) {
+    if (Array.isArray(req.data)) {
+      for (const d of req.data) {
+        generateUUIDandPropagateKeys(req.target, d, req.event)
+      }
+    } else {
+      generateUUIDandPropagateKeys(req.target, req.data, req.event)
+    }
+  }
+
+  const cmd = req.query.cmd || Object.keys(req.query)[0]
+
+  const resolved = resolveView(req.query, this.model, this)
+
+  if (resolved && resolved[cmd]._transitions?.[0].target) {
+    req.query = resolved || req.query
+    req.target = resolved?.[cmd]._transitions[0].target
+  }
+
+  return next()
+}
+
+module.exports = {
+  input
+}

package/lib/db/sqlite/ReservedWords.json

@@ -0,0 +1,149 @@
+{
+    "ABORT":1,
+    "ACTION":1,
+    "ADD":1,
+    "AFTER":1,
+    "ALL":1,
+    "ALTER":1,
+    "ALWAYS":1,
+    "ANALYZE":1,
+    "AND":1,
+    "AS":1,
+    "ASC":1,
+    "ATTACH":1,
+    "AUTOINCREMENT":1,
+    "BEFORE":1,
+    "BEGIN":1,
+    "BETWEEN":1,
+    "BY":1,
+    "CASCADE":1,
+    "CASE":1,
+    "CAST":1,
+    "CHECK":1,
+    "COLLATE":1,
+    "COLUMN":1,
+    "COMMIT":1,
+    "CONFLICT":1,
+    "CONSTRAINT":1,
+    "CREATE":1,
+    "CROSS":1,
+    "CURRENT":1,
+    "CURRENT_DATE":1,
+    "CURRENT_TIME":1,
+    "CURRENT_TIMESTAMP":1,
+    "DATABASE":1,
+    "DEFAULT":1,
+    "DEFERRABLE":1,
+    "DEFERRED":1,
+    "DELETE":1,
+    "DESC":1,
+    "DETACH":1,
+    "DISTINCT":1,
+    "DO":1,
+    "DROP":1,
+    "EACH":1,
+    "ELSE":1,
+    "END":1,
+    "ESCAPE":1,
+    "EXCEPT":1,
+    "EXCLUDE":1,
+    "EXCLUSIVE":1,
+    "EXISTS":1,
+    "EXPLAIN":1,
+    "FAIL":1,
+    "FILTER":1,
+    "FIRST":1,
+    "FOLLOWING":1,
+    "FOR":1,
+    "FOREIGN":1,
+    "FROM":1,
+    "FULL":1,
+    "GENERATED":1,
+    "GLOB":1,
+    "GROUP":1,
+    "GROUPS":1,
+    "HAVING":1,
+    "IF":1,
+    "IGNORE":1,
+    "IMMEDIATE":1,
+    "IN":1,
+    "INDEX":1,
+    "INDEXED":1,
+    "INITIALLY":1,
+    "INNER":1,
+    "INSERT":1,
+    "INSTEAD":1,
+    "INTERSECT":1,
+    "INTO":1,
+    "IS":1,
+    "ISNULL":1,
+    "JOIN":1,
+    "KEY":1,
+    "LAST":1,
+    "LEFT":1,
+    "LIKE":1,
+    "LIMIT":1,
+    "MATCH":1,
+    "MATERIALIZED":1,
+    "NATURAL":1,
+    "NO":1,
+    "NOT":1,
+    "NOTHING":1,
+    "NOTNULL":1,
+    "NULL":1,
+    "NULLS":1,
+    "OF":1,
+    "OFFSET":1,
+    "ON":1,
+    "OR":1,
+    "ORDER":1,
+    "OTHERS":1,
+    "OUTER":1,
+    "OVER":1,
+    "PARTITION":1,
+    "PLAN":1,
+    "PRAGMA":1,
+    "PRECEDING":1,
+    "PRIMARY":1,
+    "QUERY":1,
+    "RAISE":1,
+    "RANGE":1,
+    "RECURSIVE":1,
+    "REFERENCES":1,
+    "REGEXP":1,
+    "REINDEX":1,
+    "RELEASE":1,
+    "RENAME":1,
+    "REPLACE":1,
+    "RESTRICT":1,
+    "RETURNING":1,
+    "RIGHT":1,
+    "ROLLBACK":1,
+    "ROW":1,
+    "ROWS":1,
+    "SAVEPOINT":1,
+    "SELECT":1,
+    "SET":1,
+    "TABLE":1,
+    "TEMP":1,
+    "TEMPORARY":1,
+    "THEN":1,
+    "TIES":1,
+    "TO":1,
+    "TRANSACTION":1,
+    "TRIGGER":1,
+    "UNBOUNDED":1,
+    "UNION":1,
+    "UNIQUE":1,
+    "UPDATE":1,
+    "USING":1,
+    "VACUUM":1,
+    "VALUES":1,
+    "VIEW":1,
+    "VIRTUAL":1,
+    "WHEN":1,
+    "WHERE":1,
+    "WINDOW":1,
+    "WITH":1,
+    "WITHOUT":1
+}