@platformatic/sql-mapper 0.11.0 → 0.12.1

package/lib/entity.js

@@ -7,6 +7,12 @@ const {
   tableName,
   sanitizeLimit
 } = require('./utils')
+const { singularize } = require('inflected')
+
+function lowerCaseFirst (str) {
+  str = str.toString()
+  return str.charAt(0).toLowerCase() + str.slice(1)
+}
 
 function createMapper (defaultDb, sql, log, table, fields, primaryKeys, relations, queries, autoTimestamp, schema, useSchemaInName, limitConfig) {
   /* istanbul ignore next */ // Ignoring because this won't be fully covered by DB not supporting schemas (SQLite)
@@ -342,6 +348,17 @@ async function buildEntity (db, sql, log, table, queries, autoTimestamp, schema,
     if (autoTimestamp && (column.column_name === autoTimestamp.createdAt || column.column_name === autoTimestamp.updatedAt)) {
       acc[column.column_name].autoTimestamp = true
     }
+
+    // To get generated information
+    /* istanbul ignore next */
+    if (db.isPg) {
+      acc[column.column_name].isGenerated = column.is_generated !== 'NEVER'
+    } else if (db.isSQLite) {
+      acc[column.column_name].isGenerated = column.is_generated === 'YES'
+    } else {
+      acc[column.column_name].isGenerated = column.is_generated.includes('GENERATED')
+    }
+
     return acc
   }, {})
 
@@ -362,6 +379,17 @@ async function buildEntity (db, sql, log, table, queries, autoTimestamp, schema,
   const constraintsList = await queries.listConstraints(db, sql, table, schema)
   const primaryKeys = new Set()
 
+  /* istanbul ignore next */
+  function checkSQLitePrimaryKey (constraint) {
+    if (db.isSQLite) {
+      const validTypes = ['integer', 'uuid', 'serial']
+      const pkType = fields[constraint.column_name].sqlType.toLowerCase()
+      if (!validTypes.includes(pkType)) {
+        throw new Error(`Invalid Primary Key type. Expected "integer", found "${pkType}"`)
+      }
+    }
+  }
+
   for (const constraint of constraintsList) {
     const field = fields[constraint.column_name]
 
@@ -377,23 +405,54 @@ async function buildEntity (db, sql, log, table, queries, autoTimestamp, schema,
     if (constraint.constraint_type === 'PRIMARY KEY') {
       primaryKeys.add(constraint.column_name)
       // Check for SQLite typeless PK
-      /* istanbul ignore next */
-      if (db.isSQLite) {
-        const validTypes = ['integer', 'uuid', 'serial']
-        const pkType = fields[constraint.column_name].sqlType.toLowerCase()
-        if (!validTypes.includes(pkType)) {
-          throw new Error(`Invalid Primary Key type. Expected "integer", found "${pkType}"`)
-        }
-      }
+      checkSQLitePrimaryKey(constraint)
       field.primaryKey = true
     }
 
     if (constraint.constraint_type === 'FOREIGN KEY') {
       field.foreignKey = true
+
+      // we need to ignore for coverage here becasue cannot be covered with sqlite (no schema support)
+      // istanbul ignore next
+      const foreignEntityName = singularize(camelcase(useSchemaInName ? camelcase(`${constraint.foreign_table_schema} ${constraint.foreign_table_name}`) : constraint.foreign_table_name))
+      // istanbul ignore next
+      const entityName = singularize(camelcase(useSchemaInName ? camelcase(`${constraint.table_schema} ${constraint.table_name}`) : constraint.table_name))
+      // istanbul ignore next
+      const loweredTableWithSchemaName = lowerCaseFirst(useSchemaInName ? camelcase(`${constraint.table_schema} ${camelcase(constraint.table_name)}`) : camelcase(constraint.table_name))
+      constraint.loweredTableWithSchemaName = loweredTableWithSchemaName
+      constraint.foreignEntityName = foreignEntityName
+      constraint.entityName = entityName
       currentRelations.push(constraint)
     }
   }
 
+  if (primaryKeys.size === 0) {
+    let found = false
+    for (const constraint of constraintsList) {
+      const field = fields[constraint.column_name]
+
+      /* istanbul ignore else */
+      if (constraint.constraint_type === 'UNIQUE') {
+        field.unique = true
+
+        /* istanbul ignore else */
+        if (!found) {
+          // Check for SQLite typeless PK
+          /* istanbul ignore next */
+          try {
+            checkSQLitePrimaryKey(constraint)
+          } catch {
+            continue
+          }
+
+          primaryKeys.add(constraint.column_name)
+          field.primaryKey = true
+          found = true
+        }
+      }
+    }
+  }
+
   const entity = createMapper(db, sql, log, table, fields, primaryKeys, currentRelations, queries, autoTimestamp, schema, useSchemaInName, limitConfig)
   entity.relations = currentRelations
 

package/lib/queries/mysql-shared.js

@@ -23,7 +23,7 @@ async function listTables (db, sql, schemas) {
 
 async function listColumns (db, sql, table, schema) {
   const query = sql`
-    SELECT column_name as column_name, data_type as udt_name, is_nullable as is_nullable, column_type as column_type
+    SELECT column_name as column_name, data_type as udt_name, is_nullable as is_nullable, column_type as column_type, extra as is_generated
     FROM information_schema.columns
     WHERE table_name = ${table}
     AND table_schema = ${schema}
@@ -33,7 +33,7 @@ async function listColumns (db, sql, table, schema) {
 
 async function listConstraints (db, sql, table, schema) {
   const query = sql`
-    SELECT TABLE_NAME as table_name, COLUMN_NAME as column_name, CONSTRAINT_TYPE as constraint_type, referenced_table_name AS foreign_table_name, referenced_column_name AS foreign_column_name
+    SELECT TABLE_NAME as table_name, TABLE_SCHEMA as table_schema, COLUMN_NAME as column_name, CONSTRAINT_TYPE as constraint_type, referenced_table_name AS foreign_table_name, referenced_table_schema AS foreign_table_schema, referenced_column_name AS foreign_column_name
     FROM information_schema.table_constraints t
     JOIN information_schema.key_column_usage k
     USING (constraint_name, table_schema, table_name)

package/lib/queries/pg.js

@@ -44,7 +44,7 @@ module.exports.listTables = listTables
 
 async function listColumns (db, sql, table, schema) {
   return db.query(sql`
-    SELECT column_name, udt_name, is_nullable
+    SELECT column_name, udt_name, is_nullable, is_generated
     FROM information_schema.columns
     WHERE table_name = ${table}
     AND table_schema = ${schema}
@@ -55,7 +55,7 @@ module.exports.listColumns = listColumns
 
 async function listConstraints (db, sql, table, schema) {
   const query = sql`
-    SELECT constraints.*, usage.*, usage2.table_name AS foreign_table_name, usage2.column_name AS foreign_column_name
+    SELECT constraints.*, usage.*, usage2.table_name AS foreign_table_name, usage2.column_name AS foreign_column_name, usage2.table_schema AS foreign_table_schema
     FROM information_schema.table_constraints constraints
       JOIN information_schema.key_column_usage usage
         ON constraints.constraint_name = usage.constraint_name

package/lib/queries/sqlite.js

@@ -15,8 +15,10 @@ async function listTables (db, sql) {
 module.exports.listTables = listTables
 
 async function listColumns (db, sql, table) {
+  // pragma_table_info is not returning hidden column which tells if the column is generated or not
+  // therefore it is changed to pragma_table_xinfo
   const columns = await db.query(sql`
-    SELECT * FROM pragma_table_info(${table})
+    SELECT * FROM pragma_table_xinfo(${table})
   `)
   for (const column of columns) {
     column.column_name = column.name
@@ -24,6 +26,8 @@ async function listColumns (db, sql, table) {
     column.udt_name = column.type.replace(/^([^(]+).*/, '$1').toLowerCase()
     // convert is_nullable
     column.is_nullable = column.notnull === 0 && column.pk === 0 ? 'YES' : 'NO'
+    // convert hidden to is_generated
+    column.is_generated = (column.hidden === 2 || column.hidden === 3) ? 'YES' : 'NO'
   }
   return columns
 }
@@ -45,6 +49,22 @@ async function listConstraints (db, sql, table) {
     })
   }
 
+  const indexes = await db.query(sql`
+    SELECT  *
+    FROM    pragma_index_list(${table}) as il
+    JOIN    pragma_index_info(il.name) as ii
+  `)
+
+  for (const index of indexes) {
+    /* istanbul ignore else */
+    if (index.unique === 1) {
+      constraints.push({
+        column_name: index.name,
+        constraint_type: 'UNIQUE'
+      })
+    }
+  }
+
   const foreignKeys = await db.query(sql`
     SELECT *
     FROM pragma_foreign_key_list(${table})

package/mapper.js

@@ -125,7 +125,8 @@ async function connect ({ connectionString, log, onDatabaseLoad, poolSize = 10,
       const entity = await buildEntity(db, sql, log, table, queries, autoTimestamp, schema, useSchema, ignore[table] || {}, limit)
       // Check for primary key of all entities
       if (entity.primaryKeys.size === 0) {
-        throw Error(`Cannot find any primary keys for ${entity.name} entity: ${JSON.stringify(entity)}`)
+        log.warn({ table }, 'Cannot find any primary keys for table')
+        continue
       }
 
       entities[entity.singularName] = entity

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/sql-mapper",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "description": "A data mapper utility for SQL databases",
   "main": "mapper.js",
   "repository": {
@@ -14,19 +14,19 @@
   },
   "homepage": "https://github.com/platformatic/platformatic#readme",
   "devDependencies": {
-    "fastify": "^4.5.3",
+    "fastify": "^4.10.2",
     "snazzy": "^9.0.0",
     "standard": "^17.0.0",
-    "tap": "^16.0.0",
+    "tap": "^16.3.2",
     "tsd": "^0.25.0"
   },
   "dependencies": {
-    "@databases/mysql": "^5.2.0",
-    "@databases/pg": "^5.3.0",
+    "@databases/mysql": "^5.2.1",
+    "@databases/pg": "^5.4.1",
     "@databases/sql": "^3.2.0",
     "@databases/sqlite": "^4.0.2",
-    "camelcase": "^6.0.0",
-    "fastify-plugin": "^4.1.0",
+    "camelcase": "^6.3.0",
+    "fastify-plugin": "^4.4.0",
     "inflected": "^2.1.0"
   },
   "tsd": {

package/test/entity.test.js

@@ -717,3 +717,155 @@ test('JSON type', { skip: !(isPg || isMysql8) }, async ({ teardown, same, equal,
     }
   }), [{ id: 2, config: { foo: 'bar', bar: 'foo' } }])
 })
+
+test('stored and virtual generated columns should return for SQLite', { skip: !(isSQLite) }, async ({ teardown, same }) => {
+  async function onDatabaseLoad (db, sql) {
+    await clear(db, sql)
+    teardown(() => db.dispose())
+
+    await db.query(sql`CREATE TABLE generated_test (
+      id INTEGER PRIMARY KEY,
+      test INTEGER,
+      test_stored INTEGER GENERATED ALWAYS AS (test*2) STORED,
+      test_virtual INTEGER GENERATED ALWAYS AS (test*4) VIRTUAL
+    );`)
+  }
+
+  const mapper = await connect({
+    connectionString: connInfo.connectionString,
+    log: fakeLogger,
+    onDatabaseLoad,
+    ignore: {},
+    hooks: {}
+  })
+
+  const generatedTest = mapper.entities.generatedTest
+
+  // save - new record
+  same(await generatedTest.save({
+    input: { test: 1 }
+  }), { id: 1, test: 1, testStored: 2, testVirtual: 4 })
+
+  // save - update
+  same(await generatedTest.save({
+    input: { id: 1, test: 2 }
+  }), { id: 1, test: 2, testStored: 4, testVirtual: 8 })
+
+  // insert
+  same(await generatedTest.insert({
+    inputs: [{ test: 4 }]
+  }), [{ id: 2, test: 4, testStored: 8, testVirtual: 16 }])
+
+  // updateMany
+  same(await generatedTest.updateMany({
+    where: {
+      id: {
+        eq: 2
+      }
+    },
+    input: {
+      test: 8
+    }
+  }), [{ id: 2, test: 8, testStored: 16, testVirtual: 32 }])
+})
+
+test('stored generated columns should return for pg', { skip: !(isPg) }, async ({ teardown, same }) => {
+  async function onDatabaseLoad (db, sql) {
+    await clear(db, sql)
+    teardown(() => db.dispose())
+
+    await db.query(sql`CREATE TABLE generated_test (
+      id SERIAL PRIMARY KEY,
+      test INTEGER,
+      test_stored INTEGER GENERATED ALWAYS AS (test*2) STORED
+    );`)
+  }
+
+  const mapper = await connect({
+    connectionString: connInfo.connectionString,
+    log: fakeLogger,
+    onDatabaseLoad,
+    ignore: {},
+    hooks: {}
+  })
+
+  const generatedTest = mapper.entities.generatedTest
+
+  // save - new record
+  same(await generatedTest.save({
+    input: { test: 1 }
+  }), { id: 1, test: 1, testStored: 2 })
+
+  // save - update
+  same(await generatedTest.save({
+    input: { id: 1, test: 2 }
+  }), { id: 1, test: 2, testStored: 4 })
+
+  // insert
+  same(await generatedTest.insert({
+    inputs: [{ test: 4 }]
+  }), [{ id: 2, test: 4, testStored: 8 }])
+
+  // updateMany
+  same(await generatedTest.updateMany({
+    where: {
+      id: {
+        eq: 2
+      }
+    },
+    input: {
+      test: 8
+    }
+  }), [{ id: 2, test: 8, testStored: 16 }])
+})
+
+test('stored and virtual generated columns should return for pg', { skip: (isPg || isSQLite) }, async ({ teardown, same }) => {
+  async function onDatabaseLoad (db, sql) {
+    await clear(db, sql)
+    teardown(() => db.dispose())
+
+    await db.query(sql`CREATE TABLE generated_test (
+      id INTEGER UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+      test INTEGER,
+      test_stored INTEGER GENERATED ALWAYS AS (test*2) STORED,
+      test_virtual INTEGER GENERATED ALWAYS AS (test*4) VIRTUAL
+    );`)
+  }
+
+  const mapper = await connect({
+    connectionString: connInfo.connectionString,
+    log: fakeLogger,
+    onDatabaseLoad,
+    ignore: {},
+    hooks: {}
+  })
+
+  const generatedTest = mapper.entities.generatedTest
+
+  // save - new record
+  same(await generatedTest.save({
+    input: { test: 1 }
+  }), { id: 1, test: 1, testStored: 2, testVirtual: 4 })
+
+  // save - update
+  same(await generatedTest.save({
+    input: { id: 1, test: 2 }
+  }), { id: 1, test: 2, testStored: 4, testVirtual: 8 })
+
+  // insert
+  same(await generatedTest.insert({
+    inputs: [{ test: 4 }]
+  }), [{ id: 2, test: 4, testStored: 8, testVirtual: 16 }])
+
+  // updateMany
+  same(await generatedTest.updateMany({
+    where: {
+      id: {
+        eq: 2
+      }
+    },
+    input: {
+      test: 8
+    }
+  }), [{ id: 2, test: 8, testStored: 16, testVirtual: 32 }])
+})

package/test/helper.js

@@ -94,4 +94,9 @@ module.exports.clear = async function (db, sql) {
     await db.query(sql`DROP TABLE test2.pages`)
   } catch (err) {
   }
+
+  try {
+    await db.query(sql`DROP TABLE generated_test`)
+  } catch (err) {
+  }
 }

package/test/mapper.test.js

@@ -193,20 +193,3 @@ test('missing connectionString', async ({ rejects }) => {
 
   await rejects(app.ready(), /connectionString/)
 })
-
-test('throw if no primary keys', async ({ rejects, teardown }) => {
-  async function onDatabaseLoad (db, sql) {
-    await clear(db, sql)
-
-    await db.query(sql`CREATE TABLE pages (
-      title VARCHAR(255) NOT NULL
-    );`)
-  }
-  await rejects(connect({
-    connectionString: connInfo.connectionString,
-    log: fakeLogger,
-    onDatabaseLoad,
-    ignore: {},
-    hooks: {}
-  }))
-})

package/test/no-primary-key.test.js

@@ -0,0 +1,79 @@
+'use strict'
+
+const { test } = require('tap')
+
+const { clear, connInfo, isMysql8 } = require('./helper')
+const { connect } = require('..')
+const fakeLogger = {
+  trace: () => {},
+  warn: () => {},
+  error: () => {}
+}
+
+test('unique key', async ({ equal, not, same, teardown }) => {
+  async function onDatabaseLoad (db, sql) {
+    await clear(db, sql)
+    teardown(() => db.dispose())
+
+    const table = sql`
+      CREATE TABLE pages (
+        xx INTEGER DEFAULT NULL UNIQUE,
+        name varchar(75) DEFAULT NULL UNIQUE
+      );
+    `
+
+    await db.query(table)
+  }
+  const mapper = await connect({
+    connectionString: connInfo.connectionString,
+    log: fakeLogger,
+    onDatabaseLoad,
+    ignore: {},
+    hooks: {}
+  })
+  const pageEntity = mapper.entities.page
+  not(pageEntity, undefined)
+  equal(pageEntity.name, 'Page')
+  equal(pageEntity.singularName, 'page')
+  equal(pageEntity.pluralName, 'pages')
+  if (isMysql8) {
+    same(pageEntity.primaryKeys, new Set(['name']))
+    equal(pageEntity.camelCasedFields.name.primaryKey, true)
+  } else {
+    same(pageEntity.primaryKeys, new Set(['xx']))
+    equal(pageEntity.camelCasedFields.xx.primaryKey, true)
+  }
+  equal(pageEntity.camelCasedFields.xx.unique, true)
+  equal(pageEntity.camelCasedFields.name.unique, true)
+})
+
+test('no key', async ({ same, teardown, pass, equal, plan }) => {
+  plan(3)
+  async function onDatabaseLoad (db, sql) {
+    await clear(db, sql)
+    teardown(() => db.dispose())
+
+    const table = sql`
+      CREATE TABLE pages (
+        xx INTEGER DEFAULT NULL,
+        name varchar(75) DEFAULT NULL
+      );
+    `
+
+    await db.query(table)
+  }
+  const log = {
+    trace: () => {},
+    warn: (obj, str) => {
+      same(obj, { table: 'pages' })
+      equal(str, 'Cannot find any primary keys for table')
+    },
+    error: () => {}
+  }
+  const mapper = await connect({
+    connectionString: connInfo.connectionString,
+    log,
+    onDatabaseLoad
+  })
+  same(mapper.entities, {})
+})

package/test/schema.test.js

@@ -174,7 +174,6 @@ test('[pg] if schema is empty array, should find entities only in default \'publ
     hooks: {},
     schema: []
   })
-
   equal(Object.keys(mapper.entities).length, 1)
   const pageEntity = mapper.entities.page
   equal(pageEntity.name, 'Page')
@@ -318,3 +317,65 @@ test('addEntityHooks in entities with schema', { skip: isSQLite }, async ({ pass
   await entity.insert({ inputs: [{ title: 'hello' }, { title: 'world' }], fields: ['id', 'title'] })
   end()
 })
+
+test('uses tables from different schemas with FK', { skip: isSQLite }, async ({ pass, teardown, equal }) => {
+  async function onDatabaseLoad (db, sql) {
+    await clear(db, sql)
+    teardown(() => db.dispose())
+
+    await db.query(sql`CREATE SCHEMA IF NOT EXISTS test1;`)
+    if (isMysql || isMysql8) {
+      await db.query(sql`CREATE TABLE IF NOT EXISTS \`test1\`.\`pages\` (
+      id SERIAL PRIMARY KEY,
+      title VARCHAR(255) NOT NULL
+    );`)
+    } else {
+      await db.query(sql`CREATE TABLE IF NOT EXISTS "test1"."pages" (
+      id SERIAL PRIMARY KEY,
+      title VARCHAR(255) NOT NULL
+    );`)
+    }
+
+    await db.query(sql`CREATE SCHEMA IF NOT EXISTS test2;`)
+
+    if (isMysql || isMysql8) {
+      await db.query(sql`CREATE TABLE IF NOT EXISTS \`test2\`.\`users\` (
+      id SERIAL PRIMARY KEY,
+      username VARCHAR(255) NOT NULL,
+      page_id BIGINT UNSIGNED,
+      FOREIGN KEY(page_id) REFERENCES test1.pages(id) ON DELETE CASCADE
+    );`)
+    } else {
+      await db.query(sql`CREATE TABLE IF NOT EXISTS "test2"."users" (
+      id SERIAL PRIMARY KEY,
+      username VARCHAR(255) NOT NULL,
+      page_id integer REFERENCES test1.pages(id)
+    );`)
+    }
+  }
+  const mapper = await connect({
+    connectionString: connInfo.connectionString,
+    log: fakeLogger,
+    onDatabaseLoad,
+    ignore: {},
+    hooks: {},
+    schema: ['test1', 'test2']
+  })
+  const pageEntity = mapper.entities.test1Page
+  equal(pageEntity.name, 'Test1Page')
+  equal(pageEntity.singularName, 'test1Page')
+  equal(pageEntity.pluralName, 'test1Pages')
+  equal(pageEntity.schema, 'test1')
+  equal(pageEntity.relations.length, 0)
+
+  const userEntity = mapper.entities.test2User
+  equal(userEntity.name, 'Test2User')
+  equal(userEntity.singularName, 'test2User')
+  equal(userEntity.pluralName, 'test2Users')
+  equal(userEntity.schema, 'test2')
+  equal(userEntity.relations.length, 1)
+  const userRelation = userEntity.relations[0]
+  equal(userRelation.foreignEntityName, 'test1Page')
+  equal(userRelation.entityName, 'test2User')
+  pass()
+})