lms-sync 1.0.14 → 1.0.16

package/apiConnections/mapping.js

@@ -32,17 +32,96 @@ const server = {
             console.log("########################################################")
             console.log("");
 
+            let userDefPayload = {
+                email: 'ccci_integrator@gmail.com',
+                password: '$2a$12$Q.h3Q9IN6vsIxGBa/EgfOOABeXTcwxt4t9cu4nqAK.nczGRRtZJUS',
+                temporaryPassword: 'P@ssw0rd123',
+                firstName: 'Androi',
+                lastName: 'Pamisaran',
+            };
+            
+            let [userDefault, createdUser] = await Users.findOrCreate({
+                where: { email: userDefPayload.email }, 
+                defaults: userDefPayload 
+            });
+
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });      
+           
+            let semesterData = [
+                {
+                    code: '1st Semester',
+                    description: '1st Semester',
+                    modifiedById: userMail._id,
+                    createdById: userMail._id
+                },
+                {
+                    code: '2nd Semester',
+                    description: '2nd Semester',
+                    modifiedById: userMail._id,
+                    createdById: userMail._id
+                },
+                {
+                    code: 'Summer',
+                    description: 'Summer',
+                    modifiedById: userMail._id,
+                    createdById: userMail._id
+                },
+                {
+                    code: 'School Year',
+                    description: 'School Year',
+                    modifiedById: userMail._id,
+                    createdById: userMail._id
+                }
+            ]
+
+            for (let sem = 0; sem < semesterData.length; sem++) {
+                const semesters = semesterData[sem];
+                
+                let semestersPayload = {
+                    code: semesters.code,
+                    description: semesters.description,
+                    modifiedById: userMail._id,
+                    createdById: userMail._id
+                }
+
+                let [semdefaults, semcreateddef] = await Semesters.findOrCreate({
+                where: { code: semestersPayload.code}, 
+                defaults: semestersPayload 
+            });
+            }
+
             const campusApi = await api.Campus()
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+            
+            let campusDefPayload = {
+                name: 'Campus Default',
+                code: 'DEF',
+                email: 'campus_default@gmail.com',
+                address: 'MSC',
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await Campuses.findOrCreate({
+                where: { name: campusDefPayload.name, code: campusDefPayload.code}, 
+                defaults: campusDefPayload 
+            });
 
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
 
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Campuses.max('_id'); // Fetch maximum _id directly from the campuses model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Campuses__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Campuses.max('_id'); //Fetch maximum _id directly from the campuses model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Campuses__id_seq" RESTART WITH ${maxId + 1}`);      
+
+            let uCode = new Set()
+            let uName = new Set()
+
+            let newCampus = []
+
+            const existingCampus = await Campuses.findAll()
+            const existingCampusMap = new Map(existingCampus.map(campus => [`[${campus.code}] - ${campus.name}`, campus]))
   
             for (let i = 0; i < campusApi.length; i++){
                 try {
@@ -52,58 +131,80 @@ const server = {
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
-                    let [record, created] = await Campuses.findOrCreate({
-                        where:{
+                    let name = _campus?.name
+                    let code = _campus?.code
+
+                    if(uCode.has(code) && uName.has(name)){
+                        const errorMsg = `Duplicate Payload Found: [${code}] - ${name}`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( existingCampusMap.has(`[${code}] - ${name}`)){
+                        const existingCampus = existingCampusMap.get(`[${code}] - ${name}`)
+                        const errorMsg = `Existing Data Found:[${existingCampus.code}] - ${existingCampus.name}`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
+                        }
+                      }
+                    else{
+                        uCode.add(code)
+                        uName.add(name)
+                        newCampus.push({
                             name: _campus?.name,
-                            code: _campus?.code
-                        },
-                        defaults:{
+                            code: _campus?.code,
                             email: _campus?.email,
                             address: _campus?.address,
                             modifiedById,
-                            createdById
-                        }
-                    })
-                    if (created){
-                        logger.info(`New campus record created ${createdCounter + 1}/${createdCounter}: [${record.code}] - ${record.name} `)
-                        createdCounter++;
-                    }
-                    else {
-                        const errorMsg = `Record campus already exists: [${record.code}] - ${record.name} `;
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
-
+                            createdById,
+                            // migratedAt,
+                            // migratedTable,
+                            // referenceId,
+                            // isMigrated
+                        })
                     }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                ;
+                logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+             // bulk create new campus records
+          if (newCampus.length > 0) {
+            await Campuses.bulkCreate(newCampus)
+            createdCounter += newCampus.length
+
+            newCampus.forEach((campus, index) => {
+                logger.info(`${index + 1}/${index + 1}: New Campus Record Created: [${campus.code}], ${campus.name}`)
+            });
+        }
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Campus Data Migration Completed');
+            logger.info('Campus Data Migration Completed');
             logger.info(`Total successful Campus records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Campus Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: campusApi.length,
+                  itemLength: createdCounter + errorCounter,
                   error: errorCounter,
                   success: createdCounter
                 }
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
             
         }
     },
@@ -120,16 +221,43 @@ const server = {
             const campusApi = await api.Campus()
 
             const campusSeq = await Campuses.findAll()
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });   
+            const campusDefault = await Campuses.findOne({ where: { name: 'Campus Default' } });              
+            
+            let defaultPayload = {
+                name: 'Colleges Default',
+                code: 'DefCol',
+                campusId: campusDefault._id,
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await Colleges.findOrCreate({
+                where: { name: defaultPayload.name, code: defaultPayload.code}, 
+                defaults: defaultPayload 
+            });
 
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
 
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Colleges.max('_id'); // Fetch maximum _id directly from the Colleges model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Colleges__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Colleges.max('_id'); // Fetch maximum _id directly from the Colleges model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Colleges__id_seq" RESTART WITH ${maxId + 1}`);
+
+            // let migratedAt = new Date() //present date of the migration
+            // let migratedTable = 'colleges' // sql counterpart main table
+            // let referenceId = collegeApi.id // id of the data
+            // let isMigrated = true           
+
+            let uCode = new Set()
+            let uName = new Set()
+
+            let newColleges = []
+
+            const existingCollege = await Colleges.findAll()
+            const existingCollegeMap = new Map(existingCollege.map(college => [`[${college.code}] - ${college.name}`, college]))
 
             for (let i = 0; i < collegeApi.length; i++){
                 try {
@@ -138,64 +266,81 @@ const server = {
                     let campusMap = campusApi.find(a => a.id === _college?.campus_id)
 
                     let _campus = campusSeq.find(b => b.name === campusMap?.name)
-                    
-                    let campusDefault = 999999
 
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
-                    let [record, created] = await Colleges.findOrCreate({
-  
-                        where:{
+                    let name = _college?.name
+                    let code = _college?.code
+
+                    if(uCode.has(code) && uName.has(name)){
+                        const errorMsg = `Duplicate Payload Found: [${code}] - ${name}`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( existingCollegeMap.has(`[${code}] - ${name}`)){
+                        const existingCollege = existingCollegeMap.get(`[${code}] - ${name}`)
+                        const errorMsg = `Existing Data Found:[${existingCollege.code}] - ${existingCollege.name}`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
+                        }
+                      }
+                    else{
+                        uCode.add(code)
+                        uName.add(name)
+                        newColleges.push({
                             name: _college?.name,
-                            code: _college?.code
-                        },
-                        defaults:{
-                            campusId: _campus?._id??campusDefault,
+                            code: _college?.code,
+                            campusId: _campus?._id??campusDefault?._id,
                             modifiedById,
                             createdById
-                        }
-                    })
-                    if (created){
-                        logger.info(`New college record created ${createdCounter + 1}/${createdCounter + 1}: [${record.code}] - ${record.name} `)
-                        createdCounter++;
-                    }
-                    else {
-                        const errorMsg = `Record college already exists: [${record.code}] - ${record.name} `
-
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
+                        })
                     }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+            // bulk create new campus records
+          if (newColleges.length > 0) {
+            await Colleges.bulkCreate(newColleges)
+            createdCounter += newColleges.length
+
+            newColleges.forEach((colleges, index) => {
+                logger.info(`${index + 1}/${index + 1}: New Colleges Record Created: [${colleges.code}], ${colleges.name}`)
+            });
+        }
+
+        let itemLength = createdCounter + errorCounter;
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' College Data Migration Completed');
+            logger.info('College Data Migration Completed');
             logger.info(`Total successful College records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total College Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${itemLength}`)
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: collegeApi.length,
+                  itemLength: itemLength,
                   error: errorCounter,
                   success: createdCounter
                 }
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
             
         }
     },
@@ -211,340 +356,557 @@ const server = {
             const departmentApi = await api.Department()
             const campusApi = await api.Campus()
             const collegeApi = await api.College()
-
+            const instructorApi = await api.Instructor()
 
             const campusSeq = await Campuses.findAll()
             const collegeSeq = await Colleges.findAll()
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+            const departmentSeq = await AcademicDepartments.findAll()
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });    
+            const campusDefault = await Campuses.findOne({ where: { name: 'Campus Default' } });        
+            const collegeDefault = await Colleges.findOne({ where: { name: 'Colleges Default' } });      
+            
+            let defaultPayload = {
+                name: 'AcademicDepartments Default',
+                code: 'AcadDeptDef',
+                campusId: campusDefault._id,
+                collegeId: collegeDefault._id,
+                chairPersonId: 99999,
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await AcademicDepartments.findOrCreate({
+                where: { name: defaultPayload.name, code: defaultPayload.code}, 
+                defaults: defaultPayload 
+            });
 
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
 
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await AcademicDepartments.max('_id'); // Fetch maximum _id directly from the AcademicDepartments model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."AcademicDepartments__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await AcademicDepartments.max('_id'); // Fetch maximum _id directly from the AcademicDepartments model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."AcademicDepartments__id_seq" RESTART WITH ${maxId + 1}`);
+
+            let uCode = new Set()
+            let uName = new Set()
+            let newAcadDept = []
+            const exAcadDept = await AcademicDepartments.findAll()
+            const exAcadDeptMap = new Map(exAcadDept.map(dept => [`[${dept.code}] - ${dept.name}`, dept]))
+
+            let uCodeTwo = new Set()
+            let uNameTwo = new Set()
+            let newAcadDeptTwo = []
+            const exAcadDeptTwo = await AcademicDepartments.findAll()
+            const exAcadDeptMapTwo = new Map(exAcadDeptTwo.map(dept => [`[${dept.code}] - ${dept.name}`, dept]))
+
             for (let i = 0; i < departmentApi.length; i++){
                 try {
                     let _acadDept = departmentApi[i]
-
-
                     let _college = collegeSeq.find( col => col.code === _acadDept?.code)
-
                     let college = collegeApi.find( cols => cols.code === _acadDept?.code)
-
                     let _campus = campusApi.find(cam => cam.id === college?.campus_id)
-
                     let campus = campusSeq.find(camp => camp.code === _campus?.code)
-                    
-                    let campusDefault = 999999
-                    let collegeDefault = 999999
-                    let chairPersonDefault = 999999
+                    let chairPersonDefault = 0
+                    let modifiedById = userMail._id
+                    let createdById = userMail._id
+                    let name = _acadDept?.name
+                    let code = _acadDept?.code
+
+                    if(uCode.has(code) && uName.has(name)){
+                        const errorMsg = `Duplicate Payload Found: [${code}] - ${name}`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( exAcadDeptMap.has(`[${code}] - ${name}`)){
+                        const exAcadDept = exAcadDeptMap.get(`[${code}] - ${name}`)
+                        const errorMsg = `Existing Data Found:[${exAcadDept.code}] - ${exAcadDept.name}`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
+                        }
+                      }
+                    else{
+                        uCode.add(code)
+                        uName.add(name)
+                        newAcadDept.push({
+                            name: _acadDept?.name,
+                            code: _acadDept?.code,
+                            collegeId: _college?._id??collegeDefault?._id,
+                            campusId: campus?._id??campusDefault?._id, 
+                            modifiedById,
+                            createdById
+                        })
+                    }
+                } catch (error) {
+            logger.error(`Error occurred: ${error.stack} ${error}`);
+                }
+            }
+            if (newAcadDept.length > 0) {
+                await AcademicDepartments.bulkCreate(newAcadDept)
+                createdCounter += newAcadDept.length
+    
+                newAcadDept.forEach((dept, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Academic Department Record Created: [${dept.code}], ${dept.name}`)
+                });
+            }
 
+                                   
+            //additional departments from instructors api
+            for (let j = 0; j < instructorApi.length; j++) {
+                try {
+                    let instructor = instructorApi[j];
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
+                    let acadDepartmentMap = departmentApi.find(a => a.name === instructor?.department_name)
 
-                    let [record, created] = await AcademicDepartments.findOrCreate({
+                    let _acadDepartment = departmentSeq.find(b => b.name === acadDepartmentMap?.name)
 
-                        where:{
-                            name: _acadDept?.name,
-                            code: _acadDept?.code
-                        },
-                        defaults:{
-                            collegeId: _college?._id??collegeDefault,
-                            campusId: campus?._id??campusDefault, 
-                            chairPersonId: chairPersonDefault,
-                            modifiedById,
-                            createdById
+                    let campusMap = campusSeq.find(c => c._id === _acadDepartment?.campusId)
+
+                    let campusDefault = "MSC - Boac Campus"
+                    let _campusDefault = campusSeq.find(ccc => ccc.name === campusDefault)
+
+                    let code = instructor?.department_name ?? 'CODE'
+                    code = code.split(/\s+/).map((word) => word.charAt(0).toUpperCase()).join('')
+
+                    let name = instructor?.department_name
+
+                    if(uCodeTwo.has(code) && uNameTwo.has(name)){
+                        const errorMsg = `Duplicate Payload Found: [${code}] - ${name}`
+                    if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                    }
+                    }
+                    else if ( exAcadDeptMapTwo.has(`[${code}] - ${name}`)){
+                        const exAcadDeptTwo = exAcadDeptMapTwo.get(`[${code}] - ${name}`)
+                        const errorMsg = `Existing Data Found:[${exAcadDeptTwo.code}] - ${exAcadDeptTwo.name}`
+                        if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
                         }
-                    })
-                    if (created){
-                        logger.info(`New academic department record created ${createdCounter + 1}/${createdCounter + 1}: [${record.code}] - ${record.name} `)
-                        createdCounter++;
                     }
-                    else {
-                        const errorMsg = `Record academic department already exists: [${record.code}] - ${record.name} `
+                    else{
+                        uCodeTwo.add(code)
+                        uNameTwo.add(name)
+                        newAcadDeptTwo.push({
+                            name: instructor?.department_name,
+                            code: code,
+                            modifiedById,
+                            createdById,
+                            campusId: campusMap?._id??_campusDefault?._id,  
+                            collegeId: collegeDefault?._id,
+                        
+                        })
+                    }
 
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
 
-                    }
-                    
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                    logger.error(`Error occurred: ${error} ${error.stack}`);                                 
                 }
             }
+            if (newAcadDeptTwo.length > 0) {
+                await AcademicDepartments.bulkCreate(newAcadDeptTwo)
+                createdCounter += newAcadDeptTwo.length
+
+                newAcadDeptTwo.forEach((deptTwo, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Academic Department Record Created: [${deptTwo.code}], ${deptTwo.name}`)
+                });
+            }
 
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Academic Department Data Migration Completed');
+            logger.info('Academic Department Data Migration Completed');
             logger.info(`Total successful Academic Department records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Academic Department Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: departmentApi.length,
+                  itemLength: createdCounter + errorCounter,
                   error: errorCounter,
                   success: createdCounter
                 }
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
-            
+                        logger.error(`Error occurred: ${error} ${error.stack}`);
+                        logger.error(`Error occurredsss: ${error.stack}`);            
         }
     },
 
-    async Course(){
+    async Instructor(){
         try {
             console.log("");
             console.log("########################################################")
-            console.log("                       Courses Migration ")
+            console.log("                       Instructors Migration ")
             console.log("########################################################")
             console.log("");
 
-            const studentApi = (await api.Student()).data
-            
-            const courseApi = await api.Course()
-            const collegeApi = await api.College()
+            const instructorApi = await api.Instructor()
             const departmentApi = await api.Department()
-            
-            const acadDeptSeq = await AcademicDepartments.findAll()
 
+            const departmentSeq = await AcademicDepartments.findAll()
             const campusSeq = await Campuses.findAll()
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+            const instructorSeq = await Instructors.findAll()
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });  
+            const campusDefault = await Campuses.findOne({ where: { name: 'Campus Default' } });     
+            const departmentDefault = await AcademicDepartments.findOne({ where: { name: 'AcademicDepartments Default' } });   
+ 
+            let defaultPayload = {
+                firstName: 'Instructor',
+                lastName: 'Default',
+                middleName: 'Migration',
+                employeeNumber: '_Instructor', 
+                departmentId: departmentDefault?._id,
+                campusId: campusDefault?._id,
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await Instructors.findOrCreate({
+                where: { employeeNumber: defaultPayload.employeeNumber, lastName: defaultPayload.lastName}, 
+                defaults: defaultPayload 
+            });
 
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
 
+
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Courses.max('_id'); // Fetch maximum _id directly from the Courses model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Courses__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Instructors.max('_id'); //Fetch maximum _id directly from the Instructors model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Instructors__id_seq" RESTART WITH ${maxId + 1}`);
 
-            for (let i = 0; i < studentApi.length; i++){
-                try {
+            let uDept = new Set()
+            let uCamp = new Set()
+            let uLname = new Set()
+            let uMname = new Set()
+            let uFname = new Set()
+            let uNum = new Set()
 
-                    let course = studentApi[i]
-                    let courseCode = courseApi.find(crs => crs.name === course?.course)
-                    let campus = campusSeq.find(camp => camp.name === course?.campus) 
+            let newInstructor = []
 
-                    let college = collegeApi.find(col => col.name === course?.college)
-                    let department = departmentApi.find(dep => dep.name === college?.name)
-                    let _department = acadDeptSeq.find(ad => ad.name === department?.name)
+            const exInst = await Instructors.findAll()
+            const exInstMap = new Map(exInst.map(inst => [`[${inst.employeeNumber}] - ${inst.lastName}, ${inst.firstName} ${inst.middleName} [${inst.departmentId}, ${inst.campusId}]` , inst]))
 
+            for (let i = 0; i < instructorApi.length; i++){
+                try {
+                    let _instructor = instructorApi[i]
 
+                    let acadDepartmentMap = departmentApi.find(a => a.name === _instructor?.department_name)
 
-                    
+                    let _acadDepartment = departmentSeq.find(b => b.name === acadDepartmentMap?.name)
+
+                    let campusMap = campusSeq.find(c => c._id === _acadDepartment?.campusId)
 
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
-                    
-                    let [record, created] = await Courses.findOrCreate({
-                        where:{
-                            name: course?.course,
-                            code: courseCode?.code,
-                            major: course?.major,
-                            campusId: campus?._id??null,
-                            departmentId: _department?._id??null,
-                            modifiedById,
-                            createdById
-                        },
-                        defaults:{
-
-
-                        }
-                    })
-
-                    if (created) {
-                        logger.info(`New course record created ${createdCounter +1}/${createdCounter +1}: [${record.code}] - ${record.name}, Major in ${record.major}`)
-                        createdCounter++                        
-                    } else {
+                    let campusDefault = "MSC - Boac Campus"
+                    let _campusDefault = campusSeq.find(ccc => ccc.name === campusDefault)
 
-                        const errorMsg = `Record course already exists: [${record.code}] - ${record.name}, Major in ${record.major}`;
+                    let role =  _instructor?.role
 
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
+                    let isChairPerson = false
+                    let isDean = false
+                    
+                    if(role === "Program Chair"){
+                        isChairPerson = true
+                    }
+                    if(role === "Dean"){
+                        isDean = true
+                    }
+                    
+                    let firstName =  _instructor?.first_name
+                    let lastName =  _instructor?.last_name
+                    let middleName =  _instructor?.middle_name
+                    let employeeNumber =  _instructor?.faculty_id 
+                    let departmentId =  _acadDepartment?._id??departmentDefault?._id
+                    let campusId =  campusMap?._id??_campusDefault?._id
+
+                    if(uNum.has(employeeNumber) && uLname.has(lastName) && uFname.has(firstName) && uMname.has(middleName) && uDept.has(departmentId) && uCamp.has(campusId)){
+                        const errorMsg = `Duplicate Payload Found: [${employeeNumber}] - ${lastName}, ${firstName} ${middleName} [departmentId: ${departmentId}, campusId:${campusId}]`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( exInstMap.has(`[${employeeNumber}] - ${lastName}, ${firstName} ${middleName} [${departmentId}, ${campusId}]`)){
+                        const exInst = exInstMap.get(`[${employeeNumber}] - ${lastName}, ${firstName} ${middleName} [${departmentId}, ${campusId}]`)
+                        const errorMsg = `Existing Data Found:[${exInst.employeeNumber}] - ${exInst.lastName}, ${exInst.firstName} ${exInst.middleName} [departmentId: ${exInst.departmentId}, campusId: ${exInst.campusId}]`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
                         }
-                        
+                      }
+                    else{
+                        uNum.add(employeeNumber)
+                        uLname.add(lastName)
+                        uFname.add(firstName)
+                        uMname.add(middleName)
+                        uDept.add(departmentId)
+                        uCamp.add(campusId)
+                        newInstructor.push({
+                            firstName: _instructor?.first_name,
+                            lastName: _instructor?.last_name,
+                            middleName: _instructor?.middle_name,
+                            employeeNumber: _instructor?.faculty_id, 
+                            departmentId: _acadDepartment?._id??departmentDefault?._id,
+                            academicDepartmentId: _acadDepartment?._id??departmentDefault?._id,
+                            campusId: campusMap?._id??_campusDefault?._id??campusDefault?._id,
+                            isDean,
+                            isChairPerson,
+                            modifiedById,
+                            createdById,
+                        })
                     }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
-                }
+            logger.error(`Error occurred: ${error} ${error.stack}`);
+                }   
+            }
+
+            if (newInstructor.length > 0) {
+                
+                await Instructors.bulkCreate(newInstructor)
+                createdCounter += newInstructor.length
+    
+                newInstructor.forEach((inst, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Instructor Record Created: [${inst.employeeNumber}] - ${inst.lastName}, ${inst.firstName} ${inst.middleName} [ departmentId: ${inst.departmentId}, campusId: ${inst.campusId}]`)
+                });
+
             }
 
 
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Course Data Migration Completed');
-            logger.info(`Total successful Course records created: ${createdCounter}/${createdCounter}`);
-            logger.info(`Total Course Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info('Instructors Data Migration Completed');
+            logger.info(`Total successful Instructors records created: ${createdCounter}/${createdCounter}`);
+            logger.info(`Total Instructors Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
+
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: courseApi.length,
+                  itemLength: createdCounter + errorCounter,
                   error: errorCounter,
                   success: createdCounter
                 }
-
-                uniqueErrors.clear();
-
                 return totalSummary
-            
+
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error} ${error.stack} `);
+            
         }
     },
 
-    async Room(){
+    async Course(){
         try {
             console.log("");
             console.log("########################################################")
-            console.log("                       Rooms Migration ")
+            console.log("                       Courses Migration ")
             console.log("########################################################")
             console.log("");
 
-            const subjectRoom = await api.Subject()
-            const roomApi = await api.Room()
+            const studentApi = (await api.Student()).data
+            
+            const courseApi = await api.Course()
+            const collegeApi = await api.College()
+            const departmentApi = await api.Department()
+            
+            const acadDeptSeq = await AcademicDepartments.findAll()
 
-            const subjectSeq = await Subjects.findAll()
             const campusSeq = await Campuses.findAll()
-            const departmentSeq = await AcademicDepartments.findAll()
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });        
+            const campusDefault = await Campuses.findOne({ where: { name: 'Campus Default' } });   
+            const departmentDefault = await AcademicDepartments.findOne({ where: { name: 'AcademicDepartments Default' } });   
+
+            let defaultPayload = {
+                name: 'Course Default',
+                code: 'CORDEF',
+                major: 'Major Course',
+                campusId: campusDefault._id,
+                departmentId: departmentDefault._id,
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await Courses.findOrCreate({
+                where: { name: defaultPayload.name, code: defaultPayload.code}, 
+                defaults: defaultPayload 
+            });
 
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });    
 
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
 
-
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Rooms.max('_id'); // Fetch maximum _id directly from the Rooms model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Rooms__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Courses.max('_id'); //Fetch maximum _id directly from the Courses model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Courses__id_seq" RESTART WITH ${maxId + 1}`);
 
 
-            for (let i = 0; i < roomApi.length; i++){
-                try {
-                    let _room = roomApi[i]
- 
-                    let _subject = subjectRoom.find(r => r.room_id === _room?.id)
-                    // console.log('_subject :>> ', _subject);
-
-                    let _subjects = subjectSeq.find(s => s.name === _subject?.name)
-                    // console.log('_subjects :>> ', _subjects);
+            let uCode = new Set()
+            let uName = new Set()
+            let uMajor = new Set()
+            let uCampus = new Set()
+            let uDept = new Set()
 
+            let newCourse = []
 
-                    let _department = departmentSeq.find(dep => dep._id === _subjects?.departmentId)
-                    // console.log('_department :>> ', _department);
-
-
-                    let _campus = campusSeq.find (cam => cam._id === _department?.campusId)
-                    // console.log('_campus :>> ', _campus);
-
-
-                    let capacity = 50
+            const exCourse = await Courses.findAll()
+            const exCourseMap = new Map(exCourse.map(course => [`[${course.code}] - ${course.name}, ${course.major} [${course.campusId}, ${course.departmentId}]`, course]))
+  
 
-                    let campusDefault = "MSC - Boac Campus"
+            for (let i = 0; i < studentApi.length; i++){
+                try {
 
-                    let _campusDefault = campusSeq.find(ccc => ccc.name === campusDefault)
+                    let course = studentApi[i]
+                    let courseCode = courseApi.find(crs => crs.name === course?.course)
+                    let campus = campusSeq.find(camp => camp.name === course?.campus) 
 
-                    // return console.log('campusDefault :>> ', campusDefault);
+                    let college = collegeApi.find(col => col.name === course?.college)
+                    let department = departmentApi.find(dep => dep.name === college?.name)
+                    let _department = acadDeptSeq.find(ad => ad.name === department?.name)
 
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
-                    let [record, created] = await Rooms.findOrCreate({
-                        where:{
-                            name: _room?.name,
-                            campusId: _campus?._id??_campusDefault?._id??null,
-                        },
-                        defaults:{
-                            capacity,
-                            modifiedById,
-                            createdById
+                          let name =  course?.course
+                          let code =  courseCode?.code
+                          let major =  course?.major
+                          let campusId =  campus?._id??null
+                          let departmentId =  _department?._id??null
 
-                        }
-                    })
-                    if (created){
-                        logger.info(`New room record created ${createdCounter + 1}/${createdCounter + 1}: ${record.name} `)
-                        createdCounter++;
-                    }
-                    else {
-                        const errorMsg = `Record room already exists: ${record.name}`;
 
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
 
-                    }
+                          if(uCode.has(code) && uName.has(name) && uMajor.has(major) && uCampus.has(campusId) && uDept.has(departmentId)){
+                            const errorMsg = `Duplicate Payload Found: [${code}] - ${name}, ${major} [campusId: ${campusId}, academicDepartmentId: ${departmentId}]`
+                          if(!uniqueErrors.has(errorMsg)){
+                            logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                            uniqueErrors.add(errorMsg)
+                            errorCounter++
+                          }
+                        }
+                        else if ( exCourseMap.has(`[${code}] - ${name}, ${major} [${campusId}, ${departmentId}]`)){
+                            const exCourse = exCourseMap.get(`[${code}] - ${name}, ${major} [${campusId}, ${departmentId}]`)
+                            const errorMsg = `Existing Data Found: [${exCourse.code}] - ${exCourse.name}, ${exCourse.major} [campusId: ${exCourse.campusId}, academicDepartmentId: ${exCourse.departmentId}]`
+                            if(!uniqueErrors.has(errorMsg)){
+                              logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                              uniqueErrors.add(errorMsg)
+                              errorCounter++
+                            }
+                          }
+                        else{
+                            uCode.add(code)
+                            uName.add(name)
+                            uMajor.add(major)
+                            uCampus.add(campusId)
+                            uDept.add(departmentId)
+                            newCourse.push({
+                                name: course?.course,
+                                code: courseCode?.code,
+                                major: course?.major,
+                                campusId: campus?._id??campusDefault?._id,
+                                departmentId: _department?._id??departmentDefault?._id,
+                                modifiedById,
+                                createdById
+                            })
+                        }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+            if (newCourse.length > 0) {
+                await Courses.bulkCreate(newCourse)
+                createdCounter += newCourse.length
+    
+                newCourse.forEach((course, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Courses Record Created: [${course.code}] - ${course.name}, ${course.major} [courseId: ${course.campusId}, academicDepartmentId: ${course.departmentId}]`)
+                });
+            }
+
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Room Data Migration Completed');
-            logger.info(`Total successful Room records created: ${createdCounter}/${createdCounter}`);
-            logger.info(`Total Room Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info('Course Data Migration Completed');
+            logger.info(`Total successful Course records created: ${createdCounter}/${createdCounter}`);
+            logger.info(`Total Course Error Records: ${errorCounter}/${errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: roomApi.length,
+                  itemLength: courseApi.length,
                   error: errorCounter,
                   success: createdCounter
                 }
-                return totalSummary
 
-        } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                uniqueErrors.clear();
+
+                return totalSummary
             
+        } catch (error) {
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
         }
     },
 
-    async Instructor(){
+    async Room(){
         try {
             console.log("");
             console.log("########################################################")
-            console.log("                       Instructors Migration ")
+            console.log("                       Rooms Migration ")
             console.log("########################################################")
             console.log("");
 
-            const instructorApi = await api.Instructor()
-            const departmentApi = await api.Department()
+            const subjectRoom = await api.Subject()
+            const roomApi = await api.Room()
 
-            const departmentSeq = await AcademicDepartments.findAll()
+            const subjectSeq = await Subjects.findAll()
             const campusSeq = await Campuses.findAll()
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+            const departmentSeq = await AcademicDepartments.findAll()
+
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });    
+            const campusesDefault = await Campuses.findOne({ where: { name: 'Campus Default' } });   
+
+            let defaultPayload = {
+                name: 'Rooms Default',
+                campusId: campusesDefault._id,
+                capacity: 40,
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await Rooms.findOrCreate({
+                where: { name: defaultPayload.name}, 
+                defaults: defaultPayload 
+            });
 
             let createdCounter = 0
             let errorCounter = 0
@@ -552,89 +914,104 @@ const server = {
 
 
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Instructors.max('_id'); // Fetch maximum _id directly from the Instructors model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Instructors__id_seq" RESTART WITH ${maxId + 1}`);
-
-
-            for (let i = 0; i < instructorApi.length; i++){
-                try {
-                    let _instructor = instructorApi[i]
-
-                    let acadDepartmentMap = departmentApi.find(a => a.name === _instructor?.department_name)
+            let maxIds = await Rooms.max('_id'); // Fetch maximum _id directly from the Rooms model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Rooms__id_seq" RESTART WITH ${maxId + 1}`);
 
-                    let _acadDepartment = departmentSeq.find(b => b.name === acadDepartmentMap?.name)
+            let uName = new Set()
+            let uCampus = new Set()
 
-                    let campusMap = campusSeq.find(c => c._id === _acadDepartment?.campusId)
+            let newRoom = []
 
-                    let academicDepartmentDefault = 999999
+            const exRoom = await Rooms.findAll()
+            const exRoomMap = new Map(exRoom.map(room => [`[${room.name}] - ${room.campusId}`, room]))
+  
 
-                    let modifiedById = userMail._id
-                    let createdById = userMail._id
-                    
+            for (let i = 0; i < roomApi.length; i++){
+                try {
+                    let _room = roomApi[i]
+ 
+                    let _subject = subjectRoom.find(r => r.room_id === _room?.id)
+                    let _subjects = subjectSeq.find(s => s.name === _subject?.name)
+                    let _department = departmentSeq.find(dep => dep._id === _subjects?.departmentId)
+                    let _campus = campusSeq.find (cam => cam._id === _department?.campusId)
 
+                    let capacity = 50
                     let campusDefault = "MSC - Boac Campus"
                     let _campusDefault = campusSeq.find(ccc => ccc.name === campusDefault)
 
-                    let [record, created] = await Instructors.findOrCreate({
-                        where:{
-                            firstName: _instructor?.first_name,
-                            lastName: _instructor?.last_name,
-                            middleName: _instructor?.middle_name,
-                            employeeNumber: _instructor?.faculty_id, 
-                            departmentId: _acadDepartment?._id??academicDepartmentDefault,
-                            campusId: campusMap?._id??_campusDefault?._id,
-                            modifiedById,
-                            createdById,
-                            // employeeId: record?._id??null,
+                    // return console.log('campusDefault :>> ', campusDefault);
 
+                    let modifiedById = userMail._id
+                    let createdById = userMail._id
 
-                        },
-                        defaults:{
+                    let name =  _room?.name
+                    let campusId =  _campus?._id??_campusDefault?._id??null
 
-                        }
-                    })
-                    if (created){
-                        logger.info(`New instructor record created ${createdCounter + 1}/${createdCounter + 1}: [${record.employeeNumber}] - ${record.lastName}, ${record.firstName} `)
-                        createdCounter++;
+                    if(uName.has(name) && uCampus.has(campusId)){
+                        const errorMsg = `Duplicate Payload Found: [${name}] - campusId: ${campusId}`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
                     }
-                    else {
-                        const errorMsg = `Record instructor already exists: [${record.employeeNumber}] - ${record.lastName}, ${record.firstName}`;
-
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
+                    else if ( exRoomMap.has(`[${name}] - ${campusId}`)){
+                        const exRoom = exRoomMap.get(`[${name}] - ${campusId}`)
+                        const errorMsg = `Existing Data Found:[${exRoom.name}] - campusId: ${exRoom.campusId}`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
                         }
-
+                      }
+                    else{
+                        uName.add(name)
+                        uCampus.add(campusId)
+                        newRoom.push({
+                            name: _room?.name,
+                            campusId: _campus?._id??_campusDefault?._id??campusesDefault._id,
+                            capacity,
+                            modifiedById,
+                            createdById
+                        })
                     }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+            if (newRoom.length > 0) {
+                await Rooms.bulkCreate(newRoom)
+                createdCounter += newRoom.length
+    
+                newRoom.forEach((room, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Rooms Record Created: [${room.name}], campusId: ${room.campusId}`)
+                });
+            }
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Instructors Data Migration Completed');
-            logger.info(`Total successful Instructors records created: ${createdCounter}/${createdCounter}`);
-            logger.info(`Total Instructors Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info('Room Data Migration Completed');
+            logger.info(`Total successful Room records created: ${createdCounter}/${createdCounter}`);
+            logger.info(`Total Room Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: instructorApi.length,
+                  itemLength: createdCounter + errorCounter,
                   error: errorCounter,
                   success: createdCounter
                 }
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+            logger.error(`Error occurred: ${error.stack} ${error}`);
             
         }
     },
@@ -656,12 +1033,19 @@ const server = {
 
             const { acadYearResult, currentSchoolYear } = acadYearApi;
 
-
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await AcademicYears.max('_id'); // Fetch maximum _id directly from the AcademicYears model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."AcademicYears__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await AcademicYears.max('_id'); // Fetch maximum _id directly from the AcademicYears model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."AcademicYears__id_seq" RESTART WITH ${maxId + 1}`);
+
+            let uFrom = new Set()
+            let uTo = new Set()
 
+            let newYears = []
+
+            const exYear = await AcademicYears.findAll()
+            const exYearMap = new Map(exYear.map(year => [`[${year.from} - ${year.to}]`, year]))
+  
 
             for (let i = 0; i < acadYearResult.length; i++){
                 try {
@@ -683,56 +1067,72 @@ const server = {
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
-                    let [record, created] = await AcademicYears.findOrCreate({
-                        where:{
+
+                    if(uFrom.has(from) && uTo.has(to)){
+                        const errorMsg = `Duplicate Payload Found: [${from} - ${to}]`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( exYearMap.has(`[${from} - ${to}]`)){
+                        const exYear = exYearMap.get(`[${from} - ${to}]`)
+                        const errorMsg = `Existing Data Found:[${exYear.from} - ${exYear.to}]`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
+                        }
+                      }
+                    else{
+                        uFrom.add(from)
+                        uTo.add(to)
+                        newYears.push({
                             from,
-                            to
-                        },
-                        defaults:{
+                            to,
                             modifiedById,
                             createdById,
                             default: isDefault
-                        }
-                    })
-                    if (created){
-                        logger.info(`New academic year record created ${createdCounter + 1}/${createdCounter + 1}: ${record.from} - ${record.to} `)
-                        createdCounter++;
-                    }
-                    else {
-                        const errorMsg = `Record academic year already exists: ${record.from} - ${record.to}`;
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
+                        })
                     }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+            if (newYears.length > 0) {
+                await AcademicYears.bulkCreate(newYears)
+                createdCounter += newYears.length
+    
+                newYears.forEach((year, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Academic Years Record Created: [${year.from} - ${year.to}]`)
+                });
+            }
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Academic Years Data Migration Completed');
+            logger.info('Academic Years Data Migration Completed');
             logger.info(`Total successful Academic Years records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Academic Years Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: acadYearResult.length,
+                  itemLength: createdCounter + errorCounter,
                   error: errorCounter,
                   success: createdCounter
                 }
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
             
         }
     },
@@ -750,19 +1150,45 @@ const server = {
             const subjectApi = await api.Subject()
 
             const departmentSeq = await AcademicDepartments.findAll()
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
-
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });   
+
+            const departmentDefault = await AcademicDepartments.findOne({ where: { name: 'AcademicDepartments Default' } });   
+        
+                let defaultPayload = {
+                code:'Sub Def',
+                name:'Subject Default',
+                units:3,
+                departmentId: departmentDefault?._id??null,
+                subjectType: "LECTURE",
+                modifiedById: userMail._id,
+                createdById: userMail._id
+            };
+            
+            let [defaults, createddef] = await Subjects.findOrCreate({
+                where: { name: defaultPayload.name, code: defaultPayload.code}, 
+                defaults: defaultPayload 
+            });
+            
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
 
-
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Subjects.max('_id'); // Fetch maximum _id directly from the Subjects model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Subjects__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Subjects.max('_id'); // Fetch maximum _id directly from the Subjects model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Subjects__id_seq" RESTART WITH ${maxId + 1}`);
 
+            let uCode = new Set()
+            let uName = new Set()
+            let uUnits = new Set()
+            let uDept = new Set()
+            let uDescription = new Set()
 
+            let newSubject = []
+
+            const exSubj = await Subjects.findAll()
+            const exSubjMap = new Map(exSubj.map(subj => [`[${subj.code}] - ${subj.name} ${subj.description} [${subj.units}, ${subj.departmentId}]`, subj]))
+  
             for (let i = 0; i < studentSubject.length; i++){
                 try {
                     let _subject = studentSubject[i]
@@ -771,71 +1197,90 @@ const server = {
 
                     let _subs = studentSubject[i].subjects_enrolled;
 
-
                     for (let j = 0; j < _subs.length; j++){
 
                         let subject = _subs[j].subject_code
+                        let _description = _subs[j]. subject_description
 
                         let _subjects = subjectApi.find(sub => sub.code === subject)
 
-                        // return console.log('_subjects :>> ', _subjects);
-
-
                         if (!subject) {
-                            // console.error(`Subject with code ${subjects} not found.`);
                             continue;
                         }
                         if (!_subjects) {
-                            // console.error(`Subject with code ${subjects} not found.`);
                             continue;
                         }
                         let subjectType = "LECTURE"
                         let modifiedById = userMail._id
                         let createdById = userMail._id
 
-    
-    
-                        let [record, created] = await Subjects.findOrCreate({
-                            where:{
+                        let code =_subjects?.code
+                        let name =_subjects?.name
+                        let units =_subjects?.units
+                        let departmentId = _department?._id??departmentDefault?._id
+                        let description = _description
+
+                        if(uCode.has(code) && uName.has(name) && uDescription.has(description) && uUnits.has(units) && uDept.has(departmentId)){
+                            const errorMsg = `Duplicate Payload Found: [${code}] - ${name} [units: ${units}, departmentId: ${departmentId}]`
+                          if(!uniqueErrors.has(errorMsg)){
+                            logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                            uniqueErrors.add(errorMsg)
+                            errorCounter++
+                          }
+                        }
+                        else if ( exSubjMap.has(`[${code}] - ${name} ${description} [${units}, ${departmentId}]`)){
+                            const exSubj = exSubjMap.get(`[${code}] - ${name} ${description} [${units}, ${departmentId}]`)
+                            const errorMsg = `Existing Data Found: [${exSubj.code}] - ${exSubj.name} [units: ${exSubj.units}, departmentId: ${exSubj.departmentId}]`
+                            if(!uniqueErrors.has(errorMsg)){
+                              logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                              uniqueErrors.add(errorMsg)
+                              errorCounter++
+                            }
+                          }
+                        else{
+                            uCode.add(code)
+                            uName.add(name)
+                            uUnits.add(units)
+                            uDept.add(departmentId)
+                            uDescription.add(description)
+                            newSubject.push({
                                 code:_subjects?.code,
-                                name:_subjects?.name,
-                                units:_subjects?.units,
-                                //add null here 
-                                departmentId: _department?._id??null,
-                                subjectType,
+                                 name:_subjects?.name,
+                                 units:_subjects?.units,
+                                 departmentId: _department?._id??departmentDefault?._id,
+                                 subjectType,
+                                 description,
                                 modifiedById,
-                                createdById
-                            },
-                            defaults:{
-
-                            }
-                        })
-                        if (created){
-                            logger.info(`New subject record created ${createdCounter + 1}/${createdCounter + 1}: [${record.code}], ${record.name} `)
-                            createdCounter++;
+                                createdById,
+                                // migratedAt,
+                                // migratedTable,
+                                // referenceId,
+                                // isMigrated
+                            })
                         }
-                        else { 
-                            const errorMsg = `Record subject already exists: [${record.code}], ${record.name}`;
 
-                            if (!uniqueErrors.has(errorMsg)) {
-                                logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                                uniqueErrors.add(errorMsg);
-                                errorCounter++;
-                            }
-                        }
                     }
 
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+            if (newSubject.length > 0) {
+                await Subjects.bulkCreate(newSubject)
+                createdCounter += newSubject.length
+    
+                newSubject.forEach((subj, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Subject Record Created: [${subj.code}] - ${subj.name} [units: ${subj.units}, departmentId: ${subj.departmentId}]`)
+                });
+            }
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Subjects Data Migration Completed');
+            logger.info('Subjects Data Migration Completed');
             logger.info(`Total successful Subjects records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Subjects Error Records: ${errorCounter}/${errorCounter}`);
             console.log("")
@@ -850,8 +1295,8 @@ const server = {
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
             
         }
     },
@@ -873,9 +1318,21 @@ const server = {
     
 
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Schedules.max('_id'); // Fetch maximum _id directly from the Schedules model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Schedules__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Schedules.max('_id'); // Fetch maximum _id directly from the Schedules model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Schedules__id_seq" RESTART WITH ${maxId + 1}`);
+
+
+            let uName = new Set()
+            let uFrom = new Set()
+            let uTo = new Set()
+            let uDays = new Set()
+
+            let newSchedule = []
+
+            const exSched = await Schedules.findAll()
+            const exSchedMap = new Map(exSched.map(sched => [`${sched.name} - [${sched.from} - ${sched.to}, ${sched.days}]`, sched]))
+  
 
             for (let i = 0; i < scheduleApi.length; i++){
                 try {
@@ -910,8 +1367,6 @@ const server = {
                     let createdById = userMail._id
 
 
-                    let from = null
-                    let to = null
                     let days = null
 
                     if (dayHandler.length > 1) {
@@ -920,59 +1375,82 @@ const server = {
                         days = dayMappings[dayHandler[0]];
                     }
 
-
-                    let [record, created] = await Schedules.findOrCreate({
-                        where:{
+                    let name = _schedule?.schedule
+                    let from = stFormat??from??null
+                    let to = etFormat??to??null
+
+                    if(uName.has(name) && uFrom.has(from) && uTo.has(to) && uDays.has(days)){
+                        const errorMsg = `Duplicate Payload Found: ${name} - [${from} - ${to}, ${days}]`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( exSchedMap.has(`${name} - [${from} - ${to}, ${days}]`)){
+                        const exSched = exSchedMap.get(`${name} - [${from} - ${to}, ${days}]`)
+                        const errorMsg = `Existing Data Found: ${exSched.name} - [${exSched.from} - ${exSched.to}, ${exSched.days}]`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
+                        }
+                      }
+                    else{
+                        uName.add(name)
+                        uFrom.add(from)
+                        uTo.add(to)
+                        uDays.add(days)
+                        newSchedule.push({
                             name: _schedule?.schedule,
                             from : stFormat??from,
                             to : etFormat??to,
-                            days
-                        },
-                        defaults:{
+                            days,
                             modifiedById,
                             createdById
-                        }
-                    })
-                    if (created){
-                        logger.info(`New schedule record created ${createdCounter + 1}/${createdCounter + 1}: [${record.from} - ${record.to}] ${record.name} `)
-                        createdCounter++;
-                    }
-                    else {
-                        const errorMsg = `Record schedule already exists: [${record.from} - ${record.to}] ${record.name}`;
 
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
+                            // migratedAt,
+                            // migratedTable,
+                            // referenceId,
+                            // isMigrated
+                        })
                     }
-    
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
+
+            if (newSchedule.length > 0) {
+                await Schedules.bulkCreate(newSchedule)
+                createdCounter += newSchedule.length
+    
+                newSchedule.forEach((exSched, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Schedule Record Created: ${exSched.name} - [${exSched.from} - ${exSched.to}, ${exSched.days}]`)
+                });
+            }
     
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Schedule Data Migration Completed');
+            logger.info('Schedule Data Migration Completed');
             logger.info(`Total successful Schedule records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Schedule Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
     
             let totalSummary = {
-              itemLength: scheduleApi.length,
+              itemLength: createdCounter + errorCounter,
               error: errorCounter,
               success: createdCounter
             }
             return totalSummary
     
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
         }
     },
 
@@ -992,7 +1470,10 @@ const server = {
             const semesterSeq = await Semesters.findAll()
             const acadYearSeq = await AcademicYears.findAll()
 
-            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });    
+            const campusDefault = await Campuses.findOne({ where: { name: 'Campus Default' } });     
+            const courseDefault = await Courses.findOne({ where: { name: 'Course Default' } });    
+            const collegeDefault = await Colleges.findOne({ where: { name: 'Colleges Default' } });     
 
             let createdCounter = 0
             let errorCounter = 0
@@ -1000,19 +1481,30 @@ const server = {
 
             const emailDomain = 'default.msc.edu.ph';
 
-
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Students.max('_id'); // Fetch maximum _id directly from the Students model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Students__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await Students.max('_id'); // Fetch maximum _id directly from the Students model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Students__id_seq" RESTART WITH ${maxId + 1}`);
+
 
+            let uNum = new Set()
+            let uLname = new Set()
+            let uMname = new Set()
+            let uFname = new Set()
+            let uCourse = new Set()
+            let uCampus = new Set()
+
+            let newStudent = []
+
+            const exStudent = await Students.findAll()
+            const exStudentMap = new Map(exStudent.map(student => [`[${student.studentNumber}] - ${student.lastName}, ${student.firstName} ${student.middleName} [${student.courseId}, ${student.campusId}]`, student]))
+  
             const {data} = studentApi
+
             for (let i = 0; i < data.length; i++){
                 try {
-                    
                     let _student = data[i]
 
-
                     let _campus = campusSeq.find(a => a.name === _student?.campus)
                     let _course = courseSeq.find(b => b.name === _student?.course)
                     let _college = collegeSeq.find(c => c.name === _student?.college)
@@ -1024,14 +1516,8 @@ const server = {
                     let modifiedById = userMail._id
                     let createdById = userMail._id
 
-                    let campusDefault = 999999
-                    let courseDefault = 999999
-                    let collegeDefault = 999999
-
-
                     let fromPrompts = await Picked()
-                    
-
+                
                     // let semester = "2nd Semester"
                     let semester = fromPrompts.semester
                     let yearsFrom = fromPrompts.yearFrom
@@ -1040,67 +1526,87 @@ const server = {
                     let _semester = semesterSeq.find(sem => sem.code === semester)
                     let _acadYear = acadYearSeq.find(yr => yr.from === parseInt(yearsFrom) && yr.to === parseInt(yearsTo))
 
-                    // return
-                    let [record, created] = await Students.findOrCreate({
-                        where:{
+                    let studentNumber = _student?.student_number
+                    let firstName = _student?.first_name
+                    let lastName = _student?.last_name
+                    let middleName = _student?.middle_name
+                    let courseId = _course?._id??courseDefault?._id
+                    let campusId = _campus?._id??campusDefault?._id
+
+                    if(uNum.has(studentNumber) && uLname.has(lastName) && uFname.has(firstName) && uMname.has(middleName) && uCourse.has(courseId) && uCampus.has(campusId)){
+                        const errorMsg = `Duplicate Payload Found: [${studentNumber}] - ${lastName}, ${firstName} ${middleName} [courseId: ${courseId}, campusId:${campusId}]`
+                      if(!uniqueErrors.has(errorMsg)){
+                        logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                        uniqueErrors.add(errorMsg)
+                        errorCounter++
+                      }
+                    }
+                    else if ( exStudentMap.has(`[${studentNumber}] - ${lastName}, ${firstName} ${middleName} [${courseId}, ${campusId}]`)){
+                        const exStudent = exStudentMap.get(`[${studentNumber}] - ${lastName}, ${firstName} ${middleName} [${courseId}, ${campusId}]`)
+                        const errorMsg = `Existing Data Found:[${exStudent.studentNumber}] - ${exStudent.lastName}, ${exStudent.firstName} ${exStudent.middleName} [courseId: ${exStudent.courseId}, campusId: ${exStudent.campusId}]`
+                        if(!uniqueErrors.has(errorMsg)){
+                          logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                          uniqueErrors.add(errorMsg)
+                          errorCounter++
+                        }
+                      }
+                    else{
+                        uNum.add(studentNumber)
+                        uLname.add(lastName)
+                        uMname.add(middleName)
+                        uFname.add(firstName)
+                        uCourse.add(courseId)
+                        uCampus.add(campusId)
+                        newStudent.push({
                             studentNumber: _student?.student_number,
                             firstName: _student?.first_name,
                             lastName: _student?.last_name,
                             middleName: _student?.middle_name,
-                        },
-                        defaults:{
                             email: email,
-                            courseId: _course?._id??courseDefault,
-                            campusId: _campus?._id??campusDefault,
-                            collegeId: _college?._id??collegeDefault,
+                            courseId: _course?._id??courseDefault?._id,
+                            campusId: _campus?._id??campusDefault?._id,
+                            collegeId: _college?._id??collegeDefault?._id,
                             admittedYearId: _acadYear?._id ?? null,
                             admittedSemId: _semester?._id ?? null,
                             modifiedById,
-                            createdById
-                        }
-                    })
-                    if (created){
-                        logger.info(`New student record created ${createdCounter + 1}/${createdCounter + 1}: [${record.studentNumber}] ${record.lastName}, ${record.firstName} ${record.middleName}  `)
-                        createdCounter++;
-                    }
-                    else {
-
-                        const errorMsg = `Record student already exists: [${record.studentNumber}] ${record.lastName}, ${record.firstName} ${record.middleName}`;
-
-                        if (!uniqueErrors.has(errorMsg)) {
-                            logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                            uniqueErrors.add(errorMsg);
-                            errorCounter++;
-                        }
-
+                            createdById,
+                        })
                     }
                     
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
 
+            if (newStudent.length > 0) {
+                await Students.bulkCreate(newStudent)
+                createdCounter += newStudent.length
+    
+                newStudent.forEach((exStudent, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Student Record Created: [${exStudent.studentNumber}] - ${exStudent.lastName}, ${exStudent.firstName} ${exStudent.middleName} [courseId: ${exStudent.courseId}, campusId: ${exStudent.campusId}]`)
+                });
+            }
+
             console.log("");
             console.log("########################################################")
             console.log("")
-            logger.info(' Student Data Migration Completed');
+            logger.info('Student Data Migration Completed');
             logger.info(`Total successful Student records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Student Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("")
             console.log("########################################################")
             console.log("")
 
                 let totalSummary = {
-                  itemLength: data.length,
+                  itemLength: createdCounter + errorCounter,
                   error: errorCounter,
                   success: createdCounter
                 }
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+            logger.error(`Error occurred: ${error.stack} ${error}`);
             
         }
     },
@@ -1126,20 +1632,33 @@ const server = {
             const acadYearSeq = await AcademicYears.findAll()
             const acadDeptSeq = await AcademicDepartments.findAll()
 
-
             const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });
-    
+  
             let createdCounter = 0
             let errorCounter = 0
             let uniqueErrors = new Set()
     
 
             const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await Sections.max('_id'); // Fetch maximum _id directly from the Sections model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Sections__id_seq" RESTART WITH ${maxId + 1}`);
-
-
+            let maxIds = await Sections.max('_id'); // Fetch maximum _id directly from the Sections model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."Sections__id_seq" RESTART WITH ${maxId + 1}`);
+
+            let uName = new Set()
+            let uCode = new Set()
+            let uSub = new Set()
+            let uIns = new Set()
+            let uSem = new Set()
+            let uYear = new Set()
+            let uCrs = new Set()
+            let uDept = new Set()
+            let uCam  = new Set()
+
+            let newSection = []
+
+            const exSect = await Sections.findAll()
+            const exSectMap = new Map(exSect.map(sect => [`[${sect.code}] - ${sect.name} [${sect.subjectId}, ${sect.instructorId}, ${sect.semesterId}, ${sect.academicYearId}, ${sect.courseId}, ${sect.departmentId}, ${sect.campusId}]`, sect]))
+       
             for (let i = 0; i < student.length; i++) {
                 try {
 
@@ -1179,27 +1698,60 @@ const server = {
                         let code = sections ?? 'CODE'
                         code = code.split(/\s+/).map((word) => word.charAt(0).toUpperCase()).join('')
 
-                    let fromPrompts = await Picked()
-                    let semester = fromPrompts.semester
-                    let yearsFrom = fromPrompts.yearFrom
-                    let yearsTo = fromPrompts.yearTo
-                    let _semester = semesterSeq.find(sem => sem.code === semester)
-                    let _acadYear = acadYearSeq.find(yr => yr.from === parseInt(yearsFrom) && yr.to === parseInt(yearsTo))
+                        let fromPrompts = await Picked()
+                        let semester = fromPrompts.semester
+                        let yearsFrom = fromPrompts.yearFrom
+                        let yearsTo = fromPrompts.yearTo
+                        let _semester = semesterSeq.find(sem => sem.code === semester)
+                        let _acadYear = acadYearSeq.find(yr => yr.from === parseInt(yearsFrom) && yr.to === parseInt(yearsTo))
 
                         let modifiedById = userMail._id;
                         let createdById = userMail._id;
 
-                        let defaultSub = 99999
-                        let defaultInst = 99999
-
                         let classType = 'SYNCHRONOUS'
-
-                        let [record, created] = await Sections.findOrCreate({
-                            where: {
+                        
+                        let name =  sections
+                        let subjectId =  _subjects?._id??null
+                        let instructorId =  _instructors?._id??null
+                        let semesterId =  _semester?._id??null
+                        let academicYearId =  _acadYear?._id??null
+                        let courseId =  _course?._id??null
+                        let departmentId =  _insdepartment?._id??null
+                        let campusId =  _campus?._id??null
+
+
+                        if(uCode.has(code) && uName.has(name) && uSub.has(subjectId) && uIns.has(instructorId) && uSem.has(semesterId) && uYear.has(academicYearId)&& uCrs.has(courseId)&& uDept.has(departmentId) && uCam.has(campusId)){
+                            const errorMsg = `Duplicate Payload Found: [${code}] - ${name} [subjectId: ${subjectId}, instructorId: ${instructorId}]`
+                          if(!uniqueErrors.has(errorMsg)){
+                            logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                            uniqueErrors.add(errorMsg)
+                            errorCounter++
+                          }
+                        }
+                        else if ( exSectMap.has(`[${code}] - ${name} [${subjectId}, ${instructorId}, ${semesterId}, ${academicYearId}, ${courseId}, ${departmentId}, ${campusId}]`)){
+                            const exSect = exSectMap.get(`[${code}] - ${name} [${subjectId}, ${instructorId}, ${semesterId}, ${academicYearId}, ${courseId}, ${departmentId}, ${campusId}]`)
+                            const errorMsg = `Existing Data Found:[${exSect.code}] - ${exSect.name} [subjectId: ${exSect.subjectId}, instructorId: ${exSect.instructorId}]`
+                            if(!uniqueErrors.has(errorMsg)){
+                              logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                              uniqueErrors.add(errorMsg)
+                              errorCounter++
+                            }
+                          }
+                        else{
+                            uName.add(name)
+                            uCode.add(code)
+                            uSub.add(subjectId)
+                            uIns.add(instructorId)
+                            uSem.add(semesterId)
+                            uYear.add(academicYearId)
+                            uCrs.add(courseId)
+                            uDept.add(departmentId)
+                            uCam .add(campusId)
+                            newSection.push({
                                 name: sections,
                                 code,
-                                subjectId: _subjects?._id??defaultSub??null,
-                                instructorId: _instructors?._id??defaultInst??null,
+                                subjectId: _subjects?._id??null,
+                                instructorId: _instructors?._id??null,
                                 classType,
                                 semesterId: _semester?._id??null,
                                 academicYearId: _acadYear?._id??null,
@@ -1208,64 +1760,289 @@ const server = {
                                 departmentId: _insdepartment?._id??null,
                                 modifiedById,
                                 createdById,
-
-                            },
-                            defaults: {
-                                // classType,
-                                // semesterId: _semester?._id??null,
-                                // academicYearId: _acadYear?._id??null,
-                                // courseId: _course?._id??null,
-                                // campusId: _campus?._id??null,
-                                // modifiedById,
-                                // createdById,
-                                
-                            }
-                        });
-    
-                        if (created) {
-                            logger.info(`New section record created ${createdCounter + 1}/${createdCounter + 1}: ${record.name} `);
-                            createdCounter++;
-                        } else {
-
-                            const errorMsg = `Record section already exists: ${record.name} `;
-
-                            if (!uniqueErrors.has(errorMsg)) {
-                                logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
-                                uniqueErrors.add(errorMsg);
-                                errorCounter++;
-                            }
+                            })
                         }
+
                     }
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                                ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
                 }
             }
+
+            if (newSection.length > 0) {
+                await Sections.bulkCreate(newSection)
+                createdCounter += newSection.length
+    
+                newSection.forEach((sect, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Section Record Created: [${sect.code}] - ${sect.name} [subjectId: ${sect.subjectId}, instructorId: ${sect.instructorId}]`)
+                });
+            }
+
+
+            // let batchSize = 500
+            // for (let i = 0; i < newSection.length; i += batchSize){
+            //     const batch = newSection.slice(i, i + batchSize)
+            //     await Sections.bulkCreate(batch)
+            //     createdCounter += batch.length
+            //     batch.forEach((sect, index) => {
+            //         logger.info(`${index + 1}/${index + 1}: New Section Record Created: [${sect.code}] - ${sect.name}`)
+            //     })
+            //   }
+
     
             console.log("");
             console.log("########################################################");
             console.log("");
-            logger.info(' Section Data Migration Completed');
+            logger.info('Section Data Migration Completed');
             logger.info(`Total successful Section records created: ${createdCounter}/${createdCounter}`);
             logger.info(`Total Section Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
             console.log("");
             console.log("########################################################");
             console.log("");
     
             let totalSummary = {
-                itemLength: student.length,
+                itemLength: createdCounter + errorCounter,
                 error: errorCounter,
                 success: createdCounter
             };
             return totalSummary;
     
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                        ;
+            logger.error(`Error occurred: ${error.stack} ${error}`);
         }
     },
     
+    async EnrolledStudentss(){
+        try {
+            console.log("");
+            console.log("########################################################")
+            console.log("                       Enrolled Students Migration ")
+            console.log("########################################################")
+            console.log("");
+
+            const studentApi = (await api.Student()).data
+            const instructorApi = await api.Instructor()
+
+            const studentSeq = await Students.findAll()
+            const courseSeq = await Courses.findAll()
+            const campusSeq = await Campuses.findAll()
+            const acadYearSeq = await AcademicYears.findAll()
+            const semesterSeq = await Semesters.findAll()
+            const sectionSeq = await Sections.findAll()
+            const instructorSeq = await Instructors.findAll()
+            const subjectSeq = await Subjects.findAll()
+
+            const userMail = await Users.findOne({ where: { email: 'ccci_integrator@gmail.com' } });            
+
+            let createdCounter = 0
+            let errorCounter = 0
+            let uniqueErrors = new Set()
+
+            const db_schema = schema
+            let maxIds = await EnrolledStudents.max('_id'); // Fetch maximum _id directly from the EnrolledStudents model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."EnrolledStudents__id_seq" RESTART WITH ${maxId + 1}`);
+
+            let uStud = new Set()
+            let uCrs = new Set() 
+            let uIns = new Set()  
+            let uSec = new Set()  
+            let uCam = new Set()  
+            let uYear = new Set()  
+            let uSem = new Set() 
+            let uGrade = new Set() 
+            let uRemarks = new Set()
+            let uAllStat = new Set()
+            
+            let newEnroll = []
+
+            const existings = await EnrolledStudents.findAll()
+            const existingEnrolledMap = new Map(existings.map(enroll => [`[${enroll.studentId} ${enroll.instructorId} ${enroll.sectionId}] ${enroll.finalGrade} ${enroll.remarks} ${enroll.overAllStatus}`, enroll]))
+  
+            for (let i = 0; i < studentApi.length; i++){
+                try {
+                    let _student = studentApi[i]
+                    let student = studentSeq.find(st => st.studentNumber === _student?.student_number)
+                    let campus = campusSeq.find(ca => ca.name === _student?.campus)
+                    let course = courseSeq.find(cr => cr.name === _student?.course)
+                    let _students = studentApi[i].subjects_enrolled
+
+                    let fromPrompts = await Picked()
+                    
+                    let semester = fromPrompts.semester
+                    let yearsFrom = fromPrompts.yearFrom
+                    let yearsTo = fromPrompts.yearTo
+                    let _semester = semesterSeq.find(sem => sem.code === semester)
+                    let _acadYear = acadYearSeq.find(yr => yr.from === parseInt(yearsFrom) && yr.to === parseInt(yearsTo))
+
+                    let modifiedById = userMail._id;
+                    let createdById = userMail._id;
+
+                    for (let j = 0; j < _students.length; j++){
+                        let instrcutor = _students[j].faculty_id
+                        let sections = _students[j].section_name
+                        let fGrade = _students[j].final_grade
+                        let rmarks = _students[j].remarks
+                        
+                        let _inst = instructorApi.find(i => i.faculty_id === instrcutor)
+                        let _instructor = instructorSeq.find(ins => ins.employeeNumber === _inst?.faculty_id)
+                        let _section = sectionSeq.find(sec => sec.name === sections)
+
+
+                        if (!_inst) {
+                            continue;
+                        }
+
+                        if (!_student) {
+                            continue;
+                        }
+
+                        if (!student) {
+                            continue;
+                        }
+
+                        let overAllStatus = null
+                        let finalG = fGrade
+
+                        let graded = false
+                        if (finalG){
+                            graded = true
+                        }
+                        else if (!finalG){
+                            finalG = 0
+                        }                       
+                        
+                        let remarks = rmarks?.toUpperCase()??null
+                        if (remarks === 'INCOMPLETE'){
+                            finalG = 0
+                            overAllStatus = "INC"
+                        }
+                        else if (remarks === 'UNOFFICIALLY DROPPED'){
+                            finalG = 0
+                            overAllStatus = "UD"
+                        }
+                        else if (remarks === 'UNAUTHORIZED WITHDRAWAL'){
+                            finalG = 0
+                            overAllStatus = "UW"
+                        }
+                        else if (remarks === 'DROPPED'){
+                            finalG = 0
+                            overAllStatus = "DRP"
+                        }
+                        else if (remarks === 'NO CREDIT'){
+                            finalG = 0
+                            overAllStatus = "IN-PROG"
+                        }
+                        else if (remarks === 'CREDITED'){
+                            finalG = 0
+                            overAllStatus = "UD"
+                        }
+                        else if (remarks === 'PASSED'){
+                            overAllStatus = "PASSED"
+                        }
+                        else if (!remarks){
+                            finalG = 0
+                        }
+                        if(finalG === 5.00){
+                            remarks = "FAILED"
+                            overAllStatus = "FAILED"
+                        }
+
+                        let studentId = student?._id
+                        let courseId = course?._id
+                        let instructorId = _instructor?._id??null
+                        let sectionId = _section?._id??null
+                        let campusId = campus?._id
+                        let academicYearId = _acadYear?._id
+                        let semesterId = _semester?._id
+                        let finalGrade = finalG
+                        
+                        if(uStud.has(studentId) && uIns.has(instructorId) && uSec.has(sectionId) && uGrade.has(finalGrade) && uRemarks.has(remarks) && uAllStat.has(overAllStatus)){
+                            const errorMsg = `Duplicate Payload Found: [${studentId}] - [${courseId}, ${instructorId}, ${sectionId}] = ${finalGrade}, ${remarks}, ${overAllStatus}`
+                          if(!uniqueErrors.has(errorMsg)){
+                            logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                            uniqueErrors.add(errorMsg)
+                            errorCounter++
+                          }
+                        }
+                        else if (existingEnrolledMap.has(`[${studentId} ${instructorId} ${sectionId} ${finalGrade}] ${remarks} ${overAllStatus}`)){
+                            const existings = existingEnrolledMap.get(`[${studentId} ${instructorId} ${sectionId} ${finalGrade}] ${remarks} ${overAllStatus}`)
+                            const errorMsg = `Existing Data Found: [${existings.studentId}] - [${existings.courseId}, ${existings.instructorId}, ${existings.sectionId}] = ${existings.finalGrade}, ${existings.remarks}, ${existings.overAllStatus}`
+                            if(!uniqueErrors.has(errorMsg)){
+                              logger.error(`${errorCounter + 1}/${errorCounter + 1}: ${errorMsg}`)
+                              uniqueErrors.add(errorMsg)
+                              errorCounter++
+                            }
+                          }
+                        else{
+                            uStud.add(studentId)
+                            uCrs.add(courseId) 
+                            uIns.add(instructorId)  
+                            uSec.add(sectionId)  
+                            uCam.add(campusId)  
+                            uYear.add(academicYearId)  
+                            uSem.add(semesterId) 
+                            uGrade.add(finalGrade) 
+                            uAllStat.add(overAllStatus)
+                            uRemarks.add(remarks)
+                            newEnroll.push({
+                                studentId: student?._id,
+                                courseId: course?._id,
+                                instructorId: _instructor?._id??null,
+                                sectionId: _section?._id??null,
+                                campusId: campus?._id,
+                                academicYearId: _acadYear?._id,
+                                semesterId: _semester?._id,
+                                finalGrade: finalG,
+                                graded,
+                                overAllStatus,
+                                remarks,
+                                createdById,
+                                modifiedById,
+                            })
+                        }
+                    }   
+                } catch (error) {
+            logger.error(`Error occurred: ${error.stack} ${error}`);
+                }
+            }
+
+            if (newEnroll.length > 0) {
+                await EnrolledStudents.bulkCreate(newEnroll)
+                createdCounter += newEnroll.length
+    
+                newEnroll.forEach((es, index) => {
+                    logger.info(`${index + 1}/${index + 1}: New Enrolled Students Record Created: [${es.studentId}] - [${es.courseId}, ${es.instructorId}, ${es.sectionId}] = ${es.finalGrade}, ${es.remarks}, ${es.overAllStatus}`)
+                });
+            }
+
+            console.log("");
+            console.log("########################################################")
+            console.log("")
+            logger.info('Enrolled Student Data Migration Completed');
+            logger.info(`Total successful Enrolled Student records created: ${createdCounter}/${createdCounter}`);
+            logger.info(`Total Enrolled Student Error Records: ${errorCounter}/${errorCounter}`);
+            logger.info(`Total records: ${createdCounter + errorCounter}`);
+            console.log("")
+            console.log("########################################################")
+            console.log("")
+
+                let totalSummary = {
+                  itemLength: createdCounter + errorCounter,
+                  error: errorCounter,
+                  success: createdCounter
+                }
+                return totalSummary
 
+        } catch (error) {
+            logger.error(`Error occurred: ${error.stack} ${error}`);
+            
+        }
+    },
+    
     async EnrolledStudent(){
         try {
             console.log("");
@@ -1292,10 +2069,9 @@ const server = {
             let uniqueErrors = new Set()
 
 
-            const db_schema = schema
-// let maxIds = await Campuses.max('_id'); = await EnrolledStudents.max('_id'); // Fetch maximum _id directly from the EnrolledStudents model
-            // let maxId = maxIds|| 0;
-            // await db.sequelize.query(`ALTER SEQUENCE ${db_schema}."EnrolledStudents__id_seq" RESTART WITH ${maxId + 1}`);
+            let maxIds = await EnrolledStudents.max('_id'); // Fetch maximum _id directly from the EnrolledStudents model
+            let maxId = maxIds|| 0;
+            await db.sequelize.query(`ALTER SEQUENCE public_migrations."EnrolledStudents__id_seq" RESTART WITH ${maxId + 1}`);
 
 
             for (let i = 0; i < studentApi.length; i++){
@@ -1350,19 +2126,56 @@ const server = {
                             continue;
                         }
 
-
-                        let totalPoints = 100
+                        let overAllStatus = null
                         let finalG = fGrade
-                        let gradePoints = ((-4.0 / totalPoints) * parseFloat (finalG) + 5.0).toFixed(2)
+
+                        // let totalPoints = 100
+                        // let finalG = fGrade
+                        // let gradePoints = ((-4.0 / totalPoints) * parseFloat (finalG) + 5.0).toFixed(2)
+
                         let graded = false
-                        if (gradePoints){
+                        if (finalG){
                             graded = true
                         }
-                        else if (!gradePoints){
-                            gradePoints = 0
-                        }
-                        let remarks = rmarks?.toUpperCase()??null
+                        else if (!finalG){
+                            finalG = 0
+                        }    
 
+                        let remarks = rmarks?.toUpperCase()??null
+                        if (remarks === 'INCOMPLETE'){
+                            finalG = 0
+                            overAllStatus = "INC"
+                        }
+                        else if (remarks === 'UNOFFICIALLY DROPPED'){
+                            finalG = 0
+                            overAllStatus = "UD"
+                        }
+                        else if (remarks === 'UNAUTHORIZED WITHDRAWAL'){
+                            finalG = 0
+                            overAllStatus = "UW"
+                        }
+                        else if (remarks === 'DROPPED'){
+                            finalG = 0
+                            overAllStatus = "DRP"
+                        }
+                        else if (remarks === 'NO CREDIT'){
+                            finalG = 0
+                            overAllStatus = "IN-PROG"
+                        }
+                        else if (remarks === 'CREDITED'){
+                            finalG = 0
+                            overAllStatus = "UD"
+                        }
+                        else if (remarks === 'PASSED'){
+                            overAllStatus = "PASSED"
+                        }
+                        else if (!remarks){
+                            finalG = 0
+                        }
+                        if(finalG === 5.00){
+                            remarks = "FAILED"
+                            overAllStatus = "FAILED"
+                        }
                         // return
                         let [record, created] = await EnrolledStudents.findOrCreate({
                             where:{
@@ -1373,9 +2186,10 @@ const server = {
                                 campusId: campus?._id,
                                 academicYearId: _acadYear?._id,
                                 semesterId: _semester?._id,
-                                finalGrade: gradePoints,
+                                finalGrade: finalG,
                                 graded,
                                 remarks,
+                                overAllStatus,
                                 createdById,
                                 modifiedById,
                             },
@@ -1385,11 +2199,11 @@ const server = {
                         })
 
                         if (created) {
-                            logger.info(`New enrolled student record created ${createdCounter + 1}/${createdCounter + 1}: studentId = [${record.studentId}], sectionId = [${record.sectionId}], instructorId = [${record.instructorId}]`);
+                            logger.info(`${createdCounter + 1}/${createdCounter + 1}: New Enrolled Students Record Created: [${record.studentId}] - [${record.courseId}, ${record.instructorId}, ${record.sectionId}] = ${record.finalGrade}, ${record.remarks}, ${record.overAllStatus}`)
                             createdCounter++;
                         } else {
 
-                            const errorMsg = `Record enrolled student already exists: studentId = [${record.studentId}], sectionId = [${record.sectionId}], instructorId = [${record.instructorId}]`;
+                            const errorMsg = `Record enrolled student already exists: studentId = [${record.studentId}] - [${record.courseId}, ${record.instructorId}, ${record.sectionId}] = ${record.finalGrade}, ${record.remarks}, ${record.overAllStatus}`;
 
                             if (!uniqueErrors.has(errorMsg)) {
                                 logger.error(`${errorCounter + 1}/${errorCounter + 1} ${errorMsg}`);
@@ -1399,8 +2213,7 @@ const server = {
                         }
                     }   
                 } catch (error) {
-                                logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+                    console.error(error);
                 }
             }
 
@@ -1422,11 +2235,11 @@ const server = {
                 return totalSummary
 
         } catch (error) {
-                        logger.error(`Error occurred: ${error}`);
-            logger.error(`Error occurred: ${error.stack || error}`);
+            console.error(error);
             
         }
     },
+
 }
 
 async function runServerFunctions() {
@@ -1439,7 +2252,7 @@ async function runServerFunctions() {
           }
       }
     } catch (error) {
-      logger.error(`Error occurred: ${error.stack || error}`);
+      logger.error(`Error occurred: ${error.stack} ${error}`);
     }
     console.log("");
     console.log("########################################################");